tei_pickle.py

#!/usr/bin/env python3
# -*- coding: utf-8; mode: python -*-

"""
Gather some data for further conversion steps. This is originally part of fix_tei.
"""

__version__ = "1.0"
__date__ = "20190718"
__author__ = "kthoden@mpiwg-berlin.mpg.de"

from utils.load_config import load_config

from pathlib import Path 
import os
import shutil
import argparse
import logging
import pickle
import fix_tei
from lxml import etree


BASE_DIR = Path( os.path.realpath(__file__) ).parent
SCRIPT_NAME = Path( __file__).stem

DEFAULT_INPUT_DIR = \
    Path(os.environ['INPUT_DIR'] if 'INPUT_DIR' in os.environ else './input')
DEFAULT_OUTPUT_DIR = \
    Path(os.environ['OUTPUT_DIR'] if 'OUTPUT_DIR' in os.environ else './output')


ns_tei = "http://www.tei-c.org/ns/1.0"
NS_MAP = {"t" : ns_tei}

def main(
        tei_file,
        bib_file,
        output,
):
    """The main bit"""
    xml_tree = etree.parse(str(tei_file))

    bibdata = fix_tei.parse_bibtex(bib_file)

    cited = xml_tree.xpath("//t:bibl/t:ref/@target", namespaces=NS_MAP)
    used_citekeys = [fix_tei.unescape(c[1:]) for c in cited]
    citekeys_not_in_bib = fix_tei.validate_citations(used_citekeys, bibdata)

    fix_tei.pickle_data(citekeys_not_in_bib, used_citekeys, output)
# def main ends here

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
            formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    parser.add_argument(
            "-c", "--config",
            dest="CONFIG_FILE",
            default = BASE_DIR / "config" / "eoaconvert.cfg",
            help="Name of configuration file",
            metavar="CONFIGURATION"
    )
    parser.add_argument(
            "--log-level",
            default = "INFO",
            help="log level: choose between DEBUG, INFO, WARNING, ERROR, CRITICAL"
    )
    parser.add_argument(
            "-o", "--output-dir",
            help = f"output directory. default: {DEFAULT_OUTPUT_DIR}/PUBLICATION_NAME/pickle",
            type = Path,
    )
    parser.add_argument(
            "-f", "--filename",
            default = Path("*.xml"),
            type = Path,
            help = "eoaTEI file inside PUBLICATION_DIR, or absolute path. Patterns like '*.xml' are also acceptable"
    )
    parser.add_argument(
            "-b", "--bib-file",
            default = Path("**/*.bib"),
            type = Path,
            help = "bibtex file inside PUBLICATION_DIR, or absolute path. Patterns like '**/*.bib' are also acceptable"
    )
    parser.add_argument(
            "PUBLICATION_DIR",
            help = "directory containing the publication (including resources like pictures, etc.)",
            type = Path,
    )
    parser.add_argument(
            "-!", "--overwrite",
            action = "store_true",
            default = False,
            help="overwrite files at OUTPUT_DIR"
    )
    args = parser.parse_args()

    input_dir = args.PUBLICATION_DIR
    tei_file = \
        args.filename if args.filename . is_absolute() else list( input_dir . glob( str(args.filename) ))[0]
    bib_file = \
        args.bib_file if args.bib_file . is_absolute() else list( input_dir . glob( str(args.bib_file) ))[0]

    output_dir = \
            args.output_dir if args.output_dir is not None else (DEFAULT_OUTPUT_DIR / input_dir.resolve().stem) / "pickle"
    log_dir = output_dir / "log"

    config_file = args.CONFIG_FILE
    print("The config file is ", config_file)

    if output_dir.exists():
        if args.overwrite:
            shutil.rmtree( output_dir )
        else:
            raise( Exception( f"output directory already existing: '{output_dir}'!" ) )
    if not output_dir.exists():
        os.mkdir( output_dir )

    CONFIG = load_config(
        config_file,
        args.log_level,
        (log_dir / SCRIPT_NAME) . with_suffix( ".log" ),
    )

    main(
            tei_file = tei_file,
            bib_file = bib_file,
            output = output_dir / "data.pickle",
    )
# finis