diff --git a/README.md b/README.md index 9106eb83..c185cd5c 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ more information about Compose, see the [official documentation][dc-doc]. image: postgres:9.5 environment: - POSTGRES_USER=odoo - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -93,7 +92,6 @@ more information about Compose, see the [official documentation][dc-doc]. - postgres:db environment: - ODOO_DB_USER=odoo - network_mode: bridge Once this file is created, simply move to the corresponding folder and run the following command to start Odoo: @@ -150,7 +148,6 @@ The `docker-compose.yml` should look like: environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=strong_pg_superuser_password - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -163,7 +160,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** If Odoo is behind a reverse proxy, it is also suggested to change the port published by the container (though this port is actually not opened to the @@ -205,7 +201,6 @@ The `docker-compose.yml` should look like: environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=strong_pg_superuser_password - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -221,7 +216,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** With this configuration, all the data created in the volumes will belong to the user whose UID matches the user running inside the container. @@ -341,7 +335,6 @@ The `docker-compose.yml` should look like: - POSTGRES_PASSWORD=strong_pg_superuser_password - /etc/passwd:/etc/passwd:ro user: 1001:1001 - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -358,7 +351,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** For a more dynamic UID mapping, you can use Compose [variable substitution][dk-var]. Simply export the environment variable `UID` @@ -458,7 +450,6 @@ as well as all the Git repositories it depends on, you can use the following - POSTGRES_PASSWORD=strong_pg_superuser_password - /etc/passwd:/etc/passwd:ro user: 1001:1001 - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -477,7 +468,6 @@ as well as all the Git repositories it depends on, you can use the following - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** After the repositories have been fetched, it might not be required to pull them every time the container is restarted. In that case, simply set the @@ -532,7 +522,6 @@ The `docker-compose.yml` should look like: - POSTGRES_PASSWORD=strong_pg_superuser_password - /etc/passwd:/etc/passwd:ro user: 1001:1001 - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -552,7 +541,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** If the host user has a valid SSH key under the `.ssh` folder of his home folder, he can map his `.ssh` folder instead, e.g.: diff --git a/auto_addons/addons.py b/auto_addons/addons.py index cbf8f31d..b58bbf38 100644 --- a/auto_addons/addons.py +++ b/auto_addons/addons.py @@ -3,22 +3,18 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). import os import re +import shutil +import subprocess import sys -from os import remove -from shutil import move -from urlparse import urlparse -from subprocess import call -from subprocess import check_output +import urlparse EXTRA_ADDONS_PATH = '/opt/odoo/additional_addons/' -OLD_ODOO_CONF = '/opt/odoo/etc/odoo.conf.old' +ODOO_ADDONS_PATH = '/opt/odoo/sources/odoo/addons' ODOO_CONF = '/opt/odoo/etc/odoo.conf' -ADDONS_PATH = ['/opt/odoo/sources/odoo/addons'] -DEFAULT_SCHEME = 'https://' -DEFAULT_GIT_REPO_HOSTING_SERVICE = 'github.com' +DEFAULT_SCHEME = 'https' +DEFAULT_GIT_HOSTING_SERVICE = 'github.com' DEFAULT_ORGANIZATION = 'OCA' DEPENDENCIES_FILE = 'oca_dependencies.txt' -REGEX_ADDONS_PATH = r'^addons_path\s*=\s*' class Repo(object): @@ -31,124 +27,141 @@ class Repo(object): Following the oca_dependencies.txt syntax: https://github.com/OCA/maintainer-quality-tools/blob/master/sample_files/oca_dependencies.txt """ - def __init__(self, remote_url, parent=None): - if parent: - self.parent = parent - self.branch = self.parent.branch - else: - self.parent = None - self.branch = None + def __init__(self, remote_url, fetch_dep, parent=None): self.remote_url = remote_url - self.folder_name = None + self.fetch_dep = fetch_dep + self.parent = parent + + self.scheme = DEFAULT_SCHEME + self.netloc = DEFAULT_GIT_HOSTING_SERVICE self.organization = DEFAULT_ORGANIZATION self.repository = None - self.scheme = DEFAULT_SCHEME - self.git_repo_host = DEFAULT_GIT_REPO_HOSTING_SERVICE + self.branch = parent.branch if parent else None + self.folder = None + self._parse() - def _set_branch(self, branch): - self.branch = branch if branch else self.parent.branch - - def _check_is_ssh(self, url): - # TODO For other hosting services, this part should be dynamic. - # TODO support URL like ssh://git@gitlab.domain.name:10022 - # TODO Maybe we could consider using a standard URL parser in the future. - if url.startswith('git@github.com:'): - self.scheme = 'git@' - self.git_repo_host = 'github.com:' - return True - - def _check_is_url(self, url): - return re.match(r'^https?:/{2}\w.+$', url) \ - or self._check_is_ssh(url) - - def fetch_branch_name(self): - branch_cmd = 'git --git-dir=%s/.git --work-tree=%s branch' % ( - self.path, self.path - ) - output = check_output(branch_cmd, shell=True) - for line in output.split('\n'): - if line.startswith('*'): - self._set_branch(line.replace('* ', '')) - - def _parse_organization_repo(self, remote_url): - _args = remote_url.split('/') - _len_args = len(_args) - if _len_args == 1: - # repo - self.repository = _args[0] - self.folder_name = self.repository - elif _len_args == 2: - # organization AND repo - self.organization = _args[0] - self.repository = _args[1] - self.folder_name = self.repository + @staticmethod + def _is_http(url): + # FIXME should also support following syntax (different from Git SSH): + # 'ssh://git@github.com/organization/private-repo' + return re.match(r'^https?:/{2}.+', url) + + @staticmethod + def _is_git_ssh(url): + return re.match(r'^\w+@\w+\.\w+:.+', url) + + @staticmethod + def _is_url(url): + return Repo._is_http(url) or Repo._is_git_ssh(url) + + def _parse_org_repo(self, repo): + args = repo.split('/') + + if len(args) == 1: + # Pattern: 'public-repo' + self.repository = args[0] + elif len(args) == 2: + # Pattern: 'organization/public-repo' + self.organization = args[0] + self.repository = args[1] + else: + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern #1: public-repo' + print >> sys.stderr, 'Expected pattern #2: organization/public-repo' + print >> sys.stderr, 'Actual value: %s' % repo def _parse_url(self, url): - if self.scheme == DEFAULT_SCHEME: - _url_parse = urlparse(url) - self.git_repo_host = _url_parse.netloc - _args_path = _url_parse.path.split('/') - self.organization = _args_path[1] - self.repository = _args_path[2].replace('.git', '') - self.folder_name = self.repository - elif self.scheme == 'git@': - _args = url.split(':')[1] - _args_path = _args.split('/') - self.organization = _args_path[0] - self.repository = _args_path[1].replace('.git', '') - self.folder_name = self.repository + path = None - def _parse(self): - _remote_url = self.remote_url - _args = _remote_url.split(' ') - _len_args = len(_args) - if _len_args == 1: - if self._check_is_url(_args[0]): - # url - self._parse_url(_args[0]) + if Repo._is_http(url): + # Pattern: 'https://github.com/organization/public-repo' + parsed_url = urlparse.urlparse(url) + + self.scheme = parsed_url.scheme + self.netloc = parsed_url.netloc + + if len(parsed_url.path) > 0: + path = parsed_url.path[1:] else: - # repo OR organization/repo - self._parse_organization_repo(_args[0]) - elif _len_args == 2: - if self._check_is_url(_args[0]): - # url AND branch - self._parse_url(_args[0]) - self._set_branch(_args[1]) + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern: https://github.com/organization/public-repo' + print >> sys.stderr, 'Actual value: %s' % url + return + else: + # Pattern: 'git@github.com:organization/private-repo' + self.scheme = 'ssh' + args = url.split(':') + + if len(args) == 2: + self.netloc = args[0] + path = args[1] else: - if self._check_is_url(_args[1]): - # repo AND url - self._parse_url(_args[1]) - self.folder_name = _args[0] - else: - # repo OR organization/repo AND branch - self._parse_organization_repo(_args[0]) - self._set_branch(_args[1]) - elif _len_args == 3: - if self._check_is_url(_args[1]): - # repo OR organization/repo AND url AND branch - self._parse_organization_repo(_args[0]) - self._parse_url(_args[1]) - self._set_branch(_args[2]) - self.folder_name = _args[0] + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern: git@github.com:organization/private-repo' + print >> sys.stderr, 'Actual value: %s' % url + return + + # Pattern: 'organization/repo' + args = path.split('/') + + if len(args) == 2: + self.organization = args[0] + # Repo might end with '.git' but it's optional + self.repository = args[1].replace('.git', '') + else: + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern: organization/repo' + print >> sys.stderr, 'Actual value: %s' % path + + def _parse_repo(self, repo): + if Repo._is_url(repo): + self._parse_url(repo) + else: + self._parse_org_repo(repo) + + def _parse(self): + # Clean repo (remove multiple/trailing spaces) + repo = re.sub('\s+', ' ', self.remote_url.strip()) + + # Check if folder and/or branch are provided + args = repo.split(' ') + + if len(args) == 1: + # Pattern: 'repo' + self._parse_repo(repo) + self.folder = self.repository + + if len(args) == 2: + if Repo._is_url(args[1]): + # Pattern: 'folder repo' + # This pattern is only valid if the repo is a URL + self._parse_url(args[1]) + self.folder = args[0] + else: + # Pattern: 'repo branch' + self._parse_repo(args[0]) + self.folder = self.repository + self.branch = args[1] + + elif len(args) == 3: + # Pattern: 'folder repo branch' + self._parse_repo(args[1]) + self.folder = args[0] + self.branch = args[2] @property def path(self): - return '%s%s' % (EXTRA_ADDONS_PATH, self.folder_name) + return '%s%s' % (EXTRA_ADDONS_PATH, self.folder) @property def resolve_url(self): - _resolve_url_str = '%s%s/%s/%s.git' - if self.scheme == 'git@': - _resolve_url_str = '%s%s%s/%s.git' - - _resolve_url = _resolve_url_str % ( + return '%s://%s/%s/%s.git' % ( self.scheme, - self.git_repo_host, + self.netloc, self.organization, self.repository ) - return _resolve_url @property def download_cmd(self): @@ -160,85 +173,130 @@ def download_cmd(self): cmd = 'git clone %s %s' % ( self.resolve_url, self.path ) - return cmd.split() + return cmd @property def update_cmd(self): - if self.branch: - cmd = 'git --git-dir=%s/.git --work-tree=%s pull origin %s' % ( - self.path, self.path, self.branch - ) - return cmd.split() - else: - self.fetch_branch_name() - cmd = 'git --git-dir=%s/.git --work-tree=%s pull origin %s' % ( - self.path, self.path, self.branch - ) - return cmd.split() + cmd = 'git -C %s pull origin %s' % ( + self.path, self.branch + ) + return cmd + + def _fetch_branch_name(self): + # Example of output from `git branch` command: + # 7.0 + # 7.0.1.0 + # * 7.0.1.1 + # 8.0 + # 8.0.1.0 + # 9.0 + # 9.0.1.0 + branch_cmd = 'git -C %s branch' % ( + self.path + ) - def download(self, parent=None, is_loop=False, fetch_dep=True): - if self.path in ADDONS_PATH: + # Search for the branch prefixed with '* ' + try: + found = False + output = subprocess.check_output(branch_cmd, shell=True) + for line in output.split('\n'): + if line.startswith('*'): + self.branch = line.replace('* ', '') + found = True + break + + if not found: + print >> sys.stderr, 'FATAL: cannot fetch branch name' + print >> sys.stderr, 'Path: %s' % self.path + + except Exception, e: + print >> sys.stderr, 'FATAL: cannot fetch branch name' + print >> sys.stderr, e + + def _download_dependencies(self, addons_path): + # Check if the repo contains a dependency file + filename = '%s/%s' % (self.path, DEPENDENCIES_FILE) + if not os.path.exists(filename): return + + # Download the dependencies + with open(filename) as f: + for line in f: + l = line.strip('\n').strip() + if l and not l.startswith('#'): + Repo(l, self.fetch_dep, self).download(addons_path) + + def download(self, addons_path, parent=None, is_retry=False): + + # No need to fetch a repo twice (it could also cause infinite loop) + if self.path in addons_path: + return + if os.path.exists(self.path): - if fetch_dep: - cmd = self.update_cmd - call(cmd) - else: - self.fetch_branch_name() + # Branch name is used to: + # - pull the code + # - fetch the child repos + self._fetch_branch_name() + + if self.fetch_dep: + # Perform `git pull` + print 'Pulling: %s' % self.remote_url + sys.stdout.flush() + result = subprocess.call(self.update_cmd.split()) + + if result != 0: + print >> sys.stderr, 'FATAL: cannot git pull repository' + print >> sys.stderr, 'URL: %s' % self.remote_url else: - result = call(self.download_cmd) + # Perform `git clone` + print 'Cloning: %s' % self.remote_url + sys.stdout.flush() + result = subprocess.call(self.download_cmd.split()) + if result != 0: + # Since `git clone` failed, try some workarounds if parent and parent.parent: + # Retry recursively using the ancestors' branch self.branch = parent.parent.branch self.download( + addons_path, parent=parent.parent, - is_loop=True, - fetch_dep=fetch_dep) - else: + is_retry=True) + elif not is_retry: + # Retry one last time with the default branch of the repo self.branch = None - self.download( - is_loop=True, - fetch_dep=fetch_dep) + self.download(addons_path, is_retry=True) + else: + print >> sys.stderr, 'FATAL: cannot git clone repository' + print >> sys.stderr, 'URL: %s' % self.remote_url else: - self.fetch_branch_name() + # Branch name is used to fetch the child repos + self._fetch_branch_name() - if not is_loop: - ADDONS_PATH.append(self.path) - self.download_dependency(fetch_dep) + if not is_retry: + addons_path.append(self.path) + self._download_dependencies(addons_path) - def download_dependency(self, fetch_dep=True): - filename = '%s/%s' % (self.path, DEPENDENCIES_FILE) - if not os.path.exists(filename): - return - repo_list = [] - with open(filename) as f: - for line in f: - l = line.strip('\n').strip() - if l.startswith('#') or not l: - continue - repo_list.append(Repo(l, self)) - for repo in repo_list: - repo.download( - parent=repo.parent, - fetch_dep=fetch_dep) +def write_addons_path(addons_path): + conf_file = ODOO_CONF + '.new' -def write_addons_path(): - move(ODOO_CONF, OLD_ODOO_CONF) - with open(ODOO_CONF, 'a') as target_file, open(OLD_ODOO_CONF, 'r') as source_file: - for line in source_file: - if not re.match(REGEX_ADDONS_PATH, line): - target_file.write(line) + with open(conf_file, 'a') as target, open(ODOO_CONF, 'r') as source: + # Copy all lines except for the addons_path parameter + for line in source: + if not re.match(r'^addons_path\s*=\s*', line): + target.write(line) - new_line = 'addons_path = %s' % ','.join(list(set(ADDONS_PATH))) - target_file.write(new_line) + # Append addons_path + target.write('addons_path = %s' % ','.join(addons_path)) - remove(OLD_ODOO_CONF) + shutil.move(conf_file, ODOO_CONF) def main(): fetch_dep = True remote_url = None + addons_path = [] # 1st param is FETCH_OCA_DEPENDENCIES if len(sys.argv) > 1: @@ -249,25 +307,24 @@ def main(): if len(sys.argv) > 2: remote_url = sys.argv[2] - # If the ADDONS_REPO contains a branch name, there is a space before the - # branch name so the branch name becomes the 3rd param - if len(sys.argv) > 3: - remote_url += ' ' + sys.argv[3] - if remote_url: # Only one master repo to download - Repo(remote_url).download(fetch_dep=fetch_dep) + Repo(remote_url, fetch_dep).download(addons_path) else: - # List of repos is defined in oca_dependencies.txt at the root of - # additional_addons folder, let's download them all + # List of repos defined in oca_dependencies.txt at the root of + # additional_addons folder, download them all with open(EXTRA_ADDONS_PATH + DEPENDENCIES_FILE, 'r') as f: for line in f: l = line.strip('\n').strip() - if l.startswith('#') or not l: - continue - Repo(l).download(fetch_dep=fetch_dep) + if l and not l.startswith('#'): + Repo(l, fetch_dep).download(addons_path) + + # Odoo standard addons path must be the last, in case an additional addon + # uses the same name as a standard module (e.g. Odoo EE 'web' module in v9) + addons_path.append(ODOO_ADDONS_PATH) - write_addons_path() + write_addons_path(addons_path) if __name__ == '__main__': main() +