diff --git a/eoatex2imxml.py b/eoatex2imxml.py index ce56645..449dfe8 100755 --- a/eoatex2imxml.py +++ b/eoatex2imxml.py @@ -21,7 +21,7 @@ from utils.libeoabibitem import Bibitem import utils.libeoaconvert as libeoaconvert -from utils.load_config import load_config, exec_command, check_executable, copy_dir_overwrite +from utils.load_config import load_config, exec_command, check_executable, copy_dir_overwrite, ToLog, ToFile import utils.bib2html as bib2html # imports @@ -59,8 +59,9 @@ help="Name of config file" ) parser.add_argument( - "-l", "--log-file", - default = Path("logs", SCRIPT_NAME).with_suffix(".log"), + "-l", "--log-dir", + default = Path("logs"), + # default = Path("logs", SCRIPT_NAME).with_suffix(".log"), help="logfile" ) parser.add_argument( @@ -117,7 +118,8 @@ CONFIG = load_config( CONFIG_FILE, args.log_level, - args.log_file, + (Path(args.log_dir) / SCRIPT_NAME) . with_suffix( ".log" ), + # args.log_file, ) ######################## @@ -141,9 +143,14 @@ # Paths: ############################ INPUT_DIR = Path( args.filename ).resolve().parent -INPUT_PATH_NO_EXT = args.filename +INPUT_PATH = Path( args.filename ) +if INPUT_PATH.suffix == '': + INPUT_PATH = INPUT_PATH.with_suffix( ".tex" ) +elif INPUT_PATH.suffix != ".tex": + raise( Exception( "input file matching '*.tex' expected" ) ) OUTPUT_DIR = Path( args.output_dir ) LATEX_DIR = Path ( args.latex_dir ) +LOG_DIR = Path( args.log_dir ) CONVERT_DIR = OUTPUT_DIR / "CONVERT" # CONVERT_DIR = os.getcwd() + os.path.sep + "CONVERT" @@ -151,7 +158,9 @@ DEBUG_DIR = OUTPUT_DIR / "debug" # where to output the xml file: -XML_FILE = OUTPUT_DIR / (INPUT_PATH_NO_EXT + ".xml") +XML_FILE = (OUTPUT_DIR / INPUT_PATH.name) .with_suffix( ".xml" ) + +BIB2HTML_FILENAME = "temp" ################################################# @@ -461,34 +470,35 @@ def cleanup(): def run_tralics( input_file, TRALICS_PATH_LIB, - TRALICS_LOG_PATH, + log_path, output_dir = OUTPUT_DIR, ): - fixed_tex_file_path = output_dir / Path(input_file).name + fixed_tex_file_path = output_dir / input_file.name libeoaconvert.enable_preamble( input_file, fixed_tex_file_path, "xml" ) # Convert TeX to XML via Tralics - logging.info( f"executing {TRALICS_PATH_EXEC}. log file: {TRALICS_LOG_PATH}" ) + logging.info( f"executing {TRALICS_PATH_EXEC}. log file: {log_path}" ) exec_command( - "{cmd} -log_file {log_file} -confdir {conf_dir}/tralics_conf -config {conf_dir}/tralics.tcf -utf8 -utf8output -output_dir={output_dir} -input_dir={input_dir} -input_file={input_file}".format( + "{cmd} -confdir {conf_dir}/tralics_conf -config {conf_dir}/tralics.tcf -utf8 -utf8output -output_dir={output_dir} -input_dir={input_dir} -input_file={input_file}".format( cmd = TRALICS_PATH_EXEC, - log_file = TRALICS_LOG_PATH, + # log_file = log_filename, conf_dir = TRALICS_PATH_LIB, output_dir = output_dir, - input_dir = output_dir, - input_file = input_file, + input_dir = input_file.parent, + input_file = fixed_tex_file_path, ), + output_to = ToFile( log_path ), ignore_fail = True # :-D ) # .tex -> .xml run_tralics( - input_file = INPUT_PATH_NO_EXT + '.tex', + input_file = INPUT_PATH, TRALICS_PATH_LIB = TRALICS_PATH_LIB, - TRALICS_LOG_PATH = (INPUT_PATH_NO_EXT + "-tralics.log"), + log_path = LOG_DIR / SCRIPT_NAME / (INPUT_PATH.stem + "-tralics.log"), output_dir = OUTPUT_DIR ) @@ -1177,10 +1187,9 @@ def bibl_info_from_xml( # .bib -> .json # (return json data as python dict) def write_json_bibl( - bibl_info, + bib_database, output_file, ): - (bib_type, bib_database) = bibl_info # the new solution: pandoc-citeproc # interim_bib_json_file = INPUT_PATH_NO_EXT + "-bib.json" citeproc_command = "pandoc-citeproc --bib2json %s" % bib_database + ".bib" @@ -1200,9 +1209,8 @@ def write_json_bibl( def add_bibliography_to_xml( print_bibl_element, chapter_element, - bib_database, citations_json, - tmp_citation_filename + formatted_references ): bibliography_keyword = print_bibl_element.get("keyword") if bibliography_keyword: @@ -1239,24 +1247,6 @@ def add_bibliography_to_xml( logging.info( len( citekeys ) ) csl_file = BASE_DIR / CONFIG['Auxiliaries']['CSL_FILE'] - formatted_references = bib2html.main( - bib_file = Path(bib_database).with_suffix( ".bib" ), - citekeys = citekeys, - tex_template = BASE_DIR / "bibformat" / "4ht" / "bibliography4ht.tex", - language = strLanguage, - temp_dir = tmp_citation_filename - ) - - ''' - formatted_references = libeoaconvert.format_citations( - citations_to_format, - bib_database + ".bib", - strLanguage, - tmp_citation_filename, - csl_file - )[0] - ''' - fixed_entries = libeoaconvert.fix_bib_entries(formatted_references) for entry in fixed_entries: xmlBibliographyDiv.append(entry) @@ -1274,8 +1264,8 @@ def add_bibliography_to_xml( logging.info( ".bib -> .json") citations_json = write_json_bibl( - bibl_info, - output_file = TEMP_DIR / (INPUT_PATH_NO_EXT + "-bib.json") + INPUT_DIR / bib_database, + output_file = TEMP_DIR / (INPUT_PATH.stem + "-bib.json") ) ## only for debugging (?) @@ -1290,33 +1280,58 @@ def add_bibliography_to_xml( # If Bibliography-Type is monograph search for EOAbibliography and make it all if bib_type == "monograph": - tmp_citation_filename = TEMP_DIR / "bib2html" / "used_citations-monograph" + # tmp_citation_filename = TEMP_DIR / "bib2html" / tmp if xmlTree.find(".//EOAprintbibliography") is not None: # to insert here: with keywords we can have multiple bibliographies xmlBibliography = xmlTree.find(".//EOAprintbibliography") + citekeys = xmlTree.xpath(".//citekey/text()") + formatted_bibl_info = bib2html.main( + bib_file = (INPUT_DIR / bib_database).with_suffix( ".bib" ), + citekeys = citekeys, + tex_template = BASE_DIR / "bibformat" / "4ht" / "bibliography4ht.tex", + language = strLanguage, + temp_dir = TEMP_DIR / "bib2html" / "monograph-tmp", + output_file = TEMP_DIR / "bib2html" / "used_citations-monograph.html", + log_dir = LOG_DIR / SCRIPT_NAME / "bib2html" + ) + formatted_references = formatted_bibl_info['references'] + + logging.debug( "formatted bibliography:" ) + logging.debug( etree.tostring(formatted_references) ) add_bibliography_to_xml( - xmlBibliography, - xmlTree, - bib_database = bibl_info[1], - citations_json = citations_json, - tmp_citation_filename = tmp_citation_filename + xmlBibliography, + xmlTree, + citations_json, + formatted_references ) # If Bibliography-Type is anthology search for EOAbibliography and make one per chapter elif bib_type == "anthology": for intChapterNumber, xmlChapter in enumerate(xmlChapters, start = 1): logging.debug(f"Looking at chapter {intChapterNumber}.") - tmp_citation_filename = TEMP_DIR / "bib2html" / ("used_citations-anthology-chapter_{:02d}".format(intChapterNumber)) + # tmp_citation_filename = TEMP_DIR / "bib2html" / ("used_citations-anthology-chapter_{:02d}".format(intChapterNumber)) if xmlChapter.find(".//EOAprintbibliography") is not None: xmlBibliography = xmlChapter.find(".//EOAprintbibliography") + citekeys = xmlChapter.xpath(".//citekey/text()") + formatted_bibl_info = bib2html.main( + bib_file = (INPUT_DIR / bib_database).with_suffix( ".bib" ), + citekeys = citekeys, + tex_template = BASE_DIR / "bibformat" / "4ht" / "bibliography4ht.tex", + language = strLanguage, + temp_dir = TEMP_DIR / "bib2html" / "chapter_{:02d}-tmp".format( intChapterNumber ), + output_file = TEMP_DIR / "bib2html" / "used_citations-anthology-chapter_{:02d}.html".format( intChapterNumber ), + log_dir = LOG_DIR / SCRIPT_NAME / "bib2html" + ) + formatted_citations = formatted_bibl_info['references'] + logging.debug( "formatted bibliography:" ) + logging.debug( etree.tostring(formatted_references) ) add_bibliography_to_xml( - xmlBibliography, - xmlChapter, - bib_database = bibl_info[1], - citations_json = citations_json, - tmp_citation_filename = tmp_citation_filename + xmlBibliography, + xmlChapter, + citations_json, + formatted_references ) else: @@ -1346,7 +1361,7 @@ def add_bibliography_to_xml( if bib_type == "monograph": tmp_citation_filename = "used_citations-monograph" - tmp_path_html = TEMP_DIR / (tmp_citation_filename + ".html") + tmp_path_html = (TEMP_DIR / "bib2html" / tmp_citation_filename) .with_suffix( ".html" ) with open(tmp_path_html, "r") as formatted_citations: form_cit = BeautifulSoup(formatted_citations, "html.parser") @@ -1357,15 +1372,21 @@ def add_bibliography_to_xml( if bib_type == "anthology": tmp_citation_filename = "used_citations-anthology-chapter_{:02d}".format(intChapterNumber) - tmp_path_html = TEMP_DIR / (tmp_citation_filename + ".html") - no_cite_path = TEMP_DIR / (tmp_citation_filename + "_nocitations") + tmp_path_html = (TEMP_DIR / "bib2html" / tmp_citation_filename / BIB2HTML_FILENAME) .with_suffix( ".html" ) + # no_cite_path = TEMP_DIR / "bib2html" / (tmp_citation_filename + "_nocitations") if os.path.exists(tmp_path_html): with open(tmp_path_html, "r") as formatted_citations: form_cit = BeautifulSoup(formatted_citations, "html.parser") + else: + logging.debug("no citations in this chapter") + intChapterNumber += 1 + continue + ''' elif os.path.exists(no_cite_path): logging.debug("no citations in this chapter") intChapterNumber += 1 continue + ''' counter_citations = 1 @@ -1440,7 +1461,8 @@ def add_bibliography_to_xml( # [1:-1] to remove parentheses around citations try: - citeauthoryear_value = form_cit.select("#citeauthoryear ~ p > span[data-cites='%s']" % string_citekey)[0].text[1:-1] + citeauthoryear_value = form_cit.select("#citeauthoryear ~ p > span[data-cites='%s']" % string_citekey)[0].text + # citeauthoryear_value = form_cit.select("#citeauthoryear ~ p > span[data-cites='%s']" % string_citekey)[0].text[1:-1] except IndexError: logging.error("Could not find {}. Exiting.".format(string_citekey)) sys.exit() @@ -1461,6 +1483,7 @@ def add_bibliography_to_xml( # strCitation = tmp_string3.replace("<", "<") else: strCitation = xmlCitation.find("citetext").text + logging.info( "formatted citation: {}".format( strCitation ) ) if xmlCitation.find("./page") is not None and xmlCitation.find("./page").text is not None: pages_text = libeoaconvert.gettext(xmlCitation.find("./page")) @@ -1713,8 +1736,7 @@ def add_bibliography_to_xml( pickle.dump(data_to_pickle, f, pickle.HIGHEST_PROTOCOL) grep_command = "grep -A1 -B2 'argument of \\\EOAfn' {}".format( - # INPUT_PATH_NO_EXT - OUTPUT_DIR / (INPUT_PATH_NO_EXT + "-tralics.log") + LOG_DIR / SCRIPT_NAME / (INPUT_PATH.stem + "-tralics.log") ) grep_command_arguments = shlex.split(grep_command) grep_result = subprocess.Popen(grep_command_arguments, stdout=subprocess.PIPE) diff --git a/utils/bib2html.py b/utils/bib2html.py index 7fadf0b..6865d5e 100755 --- a/utils/bib2html.py +++ b/utils/bib2html.py @@ -9,12 +9,15 @@ __date__ = "20190313" __author__ = "kthoden@mpiwg-berlin.mpg.de" +from utils.load_config import exec_command, ToFile, ToLog + import argparse import os import subprocess import shlex import logging import string +import shutil from lxml import etree from pathlib import Path @@ -102,9 +105,25 @@ def write_dummy_latex( # def write_dummy_latex ends here -def run_htlatex(tmp_filename): +def run_htlatex( + tmp_filename, + log_dir +): """Create HTML file from temporary LaTeX file""" + exec_command( + f"htlatex {tmp_filename}.tex 'xhtml,charset=utf-8' ' -cunihtf -utf8'", + output_to = ToFile( Path(log_dir) / "htlatex1.log" ) + ) + exec_command( + f"biber {tmp_filename}", + output_to = ToFile( Path(log_dir) / "biber.log" ) + ) + exec_command( + f"htlatex {tmp_filename}.tex 'xhtml,charset=utf-8' ' -cunihtf -utf8'", + output_to = ToFile( Path(log_dir) / "htlatex2.log" ) + ) +''' command = f"htlatex {tmp_filename}.tex 'xhtml,charset=utf-8' ' -cunihtf -utf8'" arguments = shlex.split(command) logging.info("Using external command htlatex with command %s" % command) @@ -119,8 +138,71 @@ def run_htlatex(tmp_filename): arguments = shlex.split(command) logging.info("Using external command htlatex with command %s" % command) subprocess.call(arguments) + ''' # def run_htlatex ends here +def create_citations(citekeys, xml_tree, style): + """Create citations""" + + if style in ["authoryear", "year"]: + pass + else: + logging.error("Unrecognized citation format, choose 'authoryear' or 'year'. Exiting") + sys.exit() + + surrounding_div = etree.fromstring(f"

cite{style}

") + + p_element = etree.Element("p") + + for citekey in citekeys: + ligated_citekey = fix_citekey(citekey) + if style == "authoryear": + format_citation = xml_tree.xpath(f"//x:h4[text() = '{ligated_citekey}']/following-sibling::x:p[2]/text()", namespaces=NS_MAP)[0].strip() + else: + format_citation = xml_tree.xpath(f"//x:h4[text() = '{ligated_citekey}']/following-sibling::x:p[3]/text()", namespaces=NS_MAP)[0].strip() + span_element = etree.fromstring(f"""{format_citation}""") + # logging.debug(citekey, ligated_citekey) + + p_element.append(span_element) + + surrounding_div.insert(1, p_element) + + return(surrounding_div) +# def create_citations ends here + + +def fix_citekey(citekey): + """Remove ligatures from citekey""" + + replacements = { + "ff" : "ff", + } + + for replacement in replacements.keys(): + ligated_citekey = citekey.replace(replacement, replacements[replacement]) + + return ligated_citekey +# def fix_citekey ends here + + +def create_reference_list(reference_list): + """Create HTML snippet for list of references""" + + entries = reference_list.xpath(f"x:dt", namespaces=NS_MAP) + + reference_div = etree.fromstring("""
""") + + for entry in entries: + entry_citekey = entry.get("id").replace("X0-", "") + reference_string = entry.xpath(f"following-sibling::x:dd[1]/x:p", namespaces=NS_MAP)[0] + formatted_reference = transform_reference(reference_string) + wrapper_div = etree.fromstring(f"""
""") + wrapper_div.append(formatted_reference) + reference_div.append(wrapper_div) + + return reference_div +# def create_reference_list ends here + def main( bib_file, @@ -128,14 +210,19 @@ def main( tex_template, language, temp_dir, - tmp_filename = "temp" + output_file, + # tmp_filename = "temp", + log_dir = "logs" ): temp_dir = Path( temp_dir ) - tmp_filename = Path( tmp_filename ) + output_file = Path( output_file ) + # tmp_filename = Path( tmp_filename ) if not temp_dir.exists(): os.makedirs( temp_dir ) + tmp_filename = Path(output_file.name) . with_suffix( "" ) + write_dummy_latex( citekeys, bib_file, @@ -145,27 +232,40 @@ def main( ) wd = Path.cwd() + log_dir = log_dir.resolve() os.chdir( temp_dir ) - run_htlatex(tmp_filename) + run_htlatex( + tmp_filename . with_suffix( "" ), + # tmp_filename, + log_dir = log_dir + ) os.chdir( wd ) - tmp_path_html = (temp_dir / tmp_filename).with_suffix( ".html" ) + tmp_path_html = temp_dir / tmp_filename . with_suffix( ".html" ) + xml_tree = etree.parse(str(tmp_path_html)) - xml_tree = etree.parse( str(tmp_path_html) ) + citation_authoryear = create_citations(citekeys, xml_tree, "authoryear") + citation_year = create_citations(citekeys, xml_tree, "year") reference_list = xml_tree.xpath(f"//x:dl[@class='thebibliography']", namespaces=NS_MAP)[0] - entries = reference_list.xpath(f"x:dt", namespaces=NS_MAP) + reference_div = create_reference_list(reference_list) - reference_div = etree.fromstring("""
""") + html_element = etree.Element("html") + html_element.insert(0, citation_authoryear) + html_element.insert(1, citation_year) + html_element.insert(2, reference_div) - for entry in entries: - entry_citekey = entry.get("id").replace("X0-", "") - reference_string = entry.xpath(f"following-sibling::x:dd[1]/x:p", namespaces=NS_MAP)[0] - formatted_reference = transform_reference(reference_string) - wrapper_div = etree.fromstring(f"""
""") - wrapper_div.append(formatted_reference) - reference_div.append(wrapper_div) - return reference_div + # print(etree.tostring(html_element)) + + tree = etree.ElementTree(html_element) + logging.info("writing '%s'" % output_file) + tree.write(str(output_file), pretty_print=True, xml_declaration=True, encoding="utf-8") + + return { + "references": reference_div, + "citation_authoryear": citation_authoryear, + "citation_year": citation_year, + } """

References

diff --git a/utils/libeoaconvert.py b/utils/libeoaconvert.py index 5b12bd5..6ffdd92 100644 --- a/utils/libeoaconvert.py +++ b/utils/libeoaconvert.py @@ -3,6 +3,8 @@ """A collection of functions for the different conversion steps""" +from utils.load_config import exec_command, ToLog, ToFile + import os import sys import subprocess @@ -211,7 +213,8 @@ def format_citations( bibdata, language, tmp_filename, - csl_file + csl_file, + log_to = ToLog ): """Return a formatted xmlstring of the used citations""" @@ -237,10 +240,10 @@ def format_citations( # citation_formatter.write("@%s\n" % entry) citation_formatter.write("\n# References\n") - command = "pandoc -o %s -t html --filter=pandoc-citeproc --bibliography=%s --csl=%s %s" % (tmp_path_html, bibdata, csl_file, tmp_path_md) - arguments = shlex.split(command) - logging.info("Using external command pandoc with command %s" % command) - subprocess.call(arguments) + exec_command( + f"pandoc -o {tmp_path_html} -t html --filter=pandoc-citeproc --bibliography={bibdata} --csl={csl_file} {tmp_path_md}", + output_to = log_to + ) with open(tmp_path_html, "r") as ding: dd = soupparser.fromstring(ding, features="html.parser") diff --git a/utils/load_config.py b/utils/load_config.py index bf396b9..32fe0e1 100644 --- a/utils/load_config.py +++ b/utils/load_config.py @@ -47,8 +47,16 @@ def exec_command( arguments = shlex.split(command) if isinstance( output_to, ToFile ): + log_file = Path( output_to.filename ) + log_dir = log_file.resolve().parent + logging.log( + getattr(logging,log_level), + f"logfile '{log_file}'", + ) + if not (log_dir.exists() and log_dir.is_dir()): + os.makedirs( log_dir ) stdout_file = open( output_to.filename, "w") - if isinstance( output_to, ToLog ): + elif isinstance( output_to, ToLog ): stdout_file = PIPE else: stdout_file = None