diff --git a/README.md b/README.md index 1df3d26..91aea49 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,50 @@ # git_deps_py -manage dependencies between git repositories (poor mans "git submodules") +manage dependencies between git repositories (lightweight "git submodules") # Status -- TODO +- experimental # Features - Standardized way how to document and manage dependencies to other repositories - For each dependency an init script can be specified - Reasonable behaviour of indirect dependencies (e.g. avoid redundant downloads and init scripts) + +# Usage Example + +consider `example` your git project with dependencies. Lets cd into it: + + $ cd example + $ ls + > dependencies.conf + +`dependencies.conf` specifies the dependencies. +Open it in your favourite editor to see how it is constructed. +Let's install the dependencies: + + $ ../bin/git_deps_py + +This will clone the git repositories specified in `dependencies.conf` into a folder (default: `./dependencies/`). +Also, for each dependency the specified versions (git hash) are "checked out" and the init script (if specified) is run. +To see default settings and learn the command line arguments, run: + + $ ../bin/git_deps_py --help + +# Installation + +## From Git + +1. clone the git repository + + $ git clone https://github.molgen.mpg.de/EditionOpenAccess/git_deps_py.git + +1. checkout the right version + + $ cd git_deps_py + $ git checkout 0.1 + +1. install into your PATH via pip + + $ pip install --user . diff --git a/bin/git_deps_py b/bin/git_deps_py new file mode 100755 index 0000000..e40328a --- /dev/null +++ b/bin/git_deps_py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + + +from pathlib import Path +from configparser import ConfigParser +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +from time import sleep +from subprocess import check_call, check_output, PIPE, STDOUT, Popen +import shlex +from os import environ + +import logging + +# these can be overwritten by cmd line args: +REPO_DIR = Path ( "." ) +STORE_DIR = REPO_DIR / "dependencies" +DEP_DIR = REPO_DIR / "dependencies" +CONFIG_FILE = REPO_DIR / "dependencies.conf" + + +repo_keys = ('uri', 'hash', 'init') + +def read_config( + config_file +): + parser = ConfigParser() + parser.read( config_file ) + for dep in parser.sections(): + for key in ('uri', 'hash'): + if key not in parser[dep]: + raise( Exception( f"error reading config file: dependency '{dep}': missing key '{key}'" ) ) + for key in parser[dep]: + if key not in repo_keys: + raise( Exception( f"error reading config file: dependency '{dep}': unknown field '{key}'" ) ) + return parser + +def get_dep( + dest_dir, + log_dir, + name, + uri, + hash, + init_script = None +): + logging.info( f"dependency: {name}" ) + + # clone, if necessary: + repo_dir = (dest_dir / name) + log_file = (log_dir / name) . with_suffix( ".log" ) + if not repo_dir . is_dir(): + exec_command( + f'git clone "{uri}" "{repo_dir}"' + ) + exec_command( + f'git checkout "{hash}"', + cwd = repo_dir + ) + + # run script, if any: + if init_script is not None: + logging.debug( f"calling init script '{init_script}'..." ) + env = environ.copy() + env['PYTHONUNBUFFERED'] = '1' + exec_command( + init_script, + cwd = repo_dir, + env = env, + output_to = ToFile( log_file ), + ) + + # if wrong version, error! + current_version = \ + check_output( + ["git", "rev-parse", "HEAD"], + cwd = repo_dir, + universal_newlines = True + ).strip() + if current_version != hash: + logging.error( f"{name}: version conflict. Needed: '{hash}', found: '{current_version}'" ) + logging.info( f"remove the repo and try again!" ) + exit(1) + +def init_logging( + log_level, # log level in the terminal + log_file, + log_level_file = logging.DEBUG + ): + + ###################### + # Set up logging # + ###################### + + log_dir = log_file.parent + if not (log_dir.exists() and log_dir.is_dir()): + log_dir . mkdir( parents=True, exist_ok=True ) + sleep( 1 ) + + # always log to file: + logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(message)s', + filename = log_file, + filemode = "w" + ) + + rootLogger = logging.getLogger() + + # set up logging to terminal: + terminal_formatter = \ + logging.Formatter( + "%(levelname)s - %(message)s" + ) + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(terminal_formatter) + consoleHandler.setLevel( log_level ) + rootLogger.addHandler(consoleHandler) + +class ToFile: + def __init__( self, filename ): + self.filename = filename + +class ToLog: + pass + +def exec_command( + command, + error_msg = "ERROR while running {command}", + cwd = None, + env = None, + output_to = ToLog(), + log_level = "INFO", + # ignore_fail = False + exit_code_ok = lambda x: x == 0, + ): + + logging.log( + getattr(logging,log_level), + f"executing '{command}'", + ) + + arguments = shlex.split(command) + + stdout_file = None + if isinstance( output_to, ToFile ): + log_file = Path( output_to.filename ) + log_dir = log_file.parent + logging.info( + f"output: '{log_file}'", + ) + if not (log_dir.exists() and log_dir.is_dir()): + os.makedirs( log_dir ) + stdout_file = open( output_to.filename, "w", 1) + elif isinstance( output_to, ToLog ): + logging.info( + f"output:", + ) + if env is not None: + env_arg = env + else: + env_arg = environ.copy() + with Popen( + arguments, + cwd = cwd, + env = env_arg, + stdout= PIPE, + stderr= STDOUT, + universal_newlines = True, + ) as proc: + for line in proc.stdout: + if isinstance( output_to, ToFile ): + stdout_file.write( line ) + elif isinstance( output_to, ToLog ): + logging.debug( "> " + line ) + + ret = proc.wait() # 0 means success + if stdout_file is not None: + stdout_file.close() + if (not exit_code_ok(ret)) and ret != 0: + logging.error( error_msg.format( command = command) ) + raise( Exception( error_msg.format( command=command ) ) ) + +if __name__ == '__main__': + parser = ArgumentParser( + description="download dependencies to other git repositories", + formatter_class = ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-c", "--config", + dest = "CONFIG_FILE", + default = CONFIG_FILE, + type = Path, + help = "config file specifying the current repositories dependencies" + ) + parser.add_argument( + "-d", "--dep-dir", + dest = "DEP_DIR", + default = DEP_DIR, + type = Path, + help = "where to the current repository expects dependencies" + ) + parser.add_argument( + "-s", "--store-dir", + dest = "STORE_DIR", + default = STORE_DIR, + type = Path, + help = "where to store dependencies" + ) + parser.add_argument( + "-v", "--verbose", + default = 'INFO', + help = "verbosity level. can be one of DEBUG, INFO, WARNING, ERROR" + ) + parser.add_argument( + "-l", "--log-dir", + default = DEP_DIR / 'log', + type = Path, + help = "config file specifying the current repositories dependencies" + ) + + + args = parser.parse_args() + CONFIG_FILE = args.CONFIG_FILE + DEP_DIR = args.DEP_DIR + STORE_DIR = args.STORE_DIR + + init_logging( + log_level = args.verbose, + log_file = args.log_dir / "git_deps_py.log" + ); + for val in ('CONFIG_FILE', 'DEP_DIR', 'STORE_DIR'): + logging.debug( + f"{val} = {globals()[val]}" + ) + + config = read_config( CONFIG_FILE ) + + for dep in config.sections(): + + get_dep( + dest_dir = STORE_DIR, + log_dir = args.log_dir, + name = dep, + uri = config[dep]['uri'], + hash = config[dep]['hash'], + init_script = + config[dep]['init'] if 'init' in config[dep] else None + ) + # if the deps are expected in DEP_DIR, but stored in STORE_DIR: + # make links DEP_DIR/* -> STORE_DIR/: + if DEP_DIR != STORE_DIR: + src = DEP_DIR / dep + if src.is_symlink(): + src.unlink() + import os + dst = STORE_DIR / dep + if not dst.is_absolute(): + dst = os.path.relpath(dst, start = src.parent ) + logging.debug( f"creating link: {src} -> {dst}") + src . symlink_to( + dst, + target_is_directory = True + ) diff --git a/example/dependencies.conf b/example/dependencies.conf new file mode 100644 index 0000000..8f16837 --- /dev/null +++ b/example/dependencies.conf @@ -0,0 +1,5 @@ +[test] + +uri = https://github.molgen.mpg.de/EditionOpenAccess/EOASkripts.git +hash = 5bf3c45a5c0b822eb425038c2ceb14722694dbe0 +init = scripts/init.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e520449 --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ + +from setuptools import setup + +setup(name='git_deps_py', + version='0.1', + description='manage dependencies to other git repositories', + classifiers=[ + 'Development Status :: experimental', + 'Programming Language :: Python :: 3.7', + ], + url='https://github.molgen.mpg.de/EditionOpenAccess/git_deps_py', + author='Samuel Gfrörer', + author_email='SamuelGfroerer@googlemail.com', + license='MIT', + scripts=['bin/git_deps_py'], + packages=[], + python_requires='>=3', + zip_safe=False) +