40095 install requirements v3

src/scielo/bin/cfg/version.txt

@@ -1,5 +1,5 @@
-4.0.095 Teste 29/03/2018
-========================
+4.0.095 29/03/2018
+==================
 - Installation with proxy
 
 - Markup XML

src/scielo/bin/xml/app_modules/__init__.py

@@ -1,37 +1,37 @@
 # coding=utf-8
 import os
 
-try:
-    from app_modules.app.config import app_texts
-except Exception as e:
-    print(e)
-    from .app.config import app_texts
-
-try:
-    from app_modules.app.config import app_caller
-except Exception as e:
-    print(e)
-    from .app.config import app_caller
 
 try:
-    from app_modules.generics import logger
-except Exception as e:
-    print(e)
-    from .generics import logger
+    from app_modules.app.config import (
+        app_texts,
+        app_caller,
+    )
+    from app_modules.generics import (
+        logger,
+        encoding,
+        system,
+    )
 
-try:
-    from app_modules.generics import encoding
 except Exception as e:
     print(e)
-    from .generics import encoding
+    from .app.config import (
+        app_texts,
+        app_caller,
+    )
+    from .generics import (
+        logger,
+        encoding,
+        system,
+    )
 
 
 THIS_FILE_LOCATION = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/')
 THIS_LOCATION = encoding.decode(THIS_FILE_LOCATION, encoding.SYS_DEFAULT_ENCODING)
 
 INVALID_APP_PATH = not str(THIS_LOCATION) == THIS_FILE_LOCATION
 
-BIN_PATH = THIS_LOCATION + '/../..'
+BIN_PATH = os.path.dirname(os.path.dirname(THIS_LOCATION))
 BIN_XML_PATH = BIN_PATH + '/xml'
 JAR_PATH = BIN_PATH + '/jar'
 RELATIVE_JAR_PATH = JAR_PATH.replace(BIN_PATH, './..')
@@ -66,9 +66,3 @@
     os.unlink(LOG_PATH+'/app_caller.log')
 except:
     pass
-
-
-appcaller = app_caller.AppCaller(
-    logger.get_logger(LOG_PATH+'/app_caller.log', 'Environment'),
-    VENV_PATH,
-    REQUIREMENTS_FILE)

src/scielo/bin/xml/app_modules/app/config/app_caller.py

@@ -6,6 +6,9 @@
 from ...generics import encoding
 
 
+# proxy_data = system.proxy_data(configuration.proxy_info)
+
+
 so, node, release, version, machine, processor = platform.uname()
 python_version = platform.python_version()
 so = so.lower()
@@ -64,16 +67,20 @@ def register_commands(self):
         return commands
 
 
+def gen_proxy_info(proxy_server_port):
+    return ProxyInfo(system.proxy_data(proxy_server_port))
+
+
 class VirtualEnv(object):
 
-    def __init__(self, logger, venv_path, requirements_filename):
+    def __init__(self, logger, venv_path):
+        encoding.debugging('VirtualEnv.VirtualEnv()')
         self.logger = logger
         self.path = venv_path
-        self.requirements = Requirements(requirements_filename)
-        self.proxy = ProxyInfo(None)
         self.setUp()
 
     def setUp(self):
+        encoding.debugging('VirtualEnv.setUp')
         if 'windows' in so:
             self.activate_filename = u'{}/Scripts/activate.bat'.format(self.path)
             self.activate_command = u'call "{}"'.format(self.activate_filename)
@@ -88,28 +95,36 @@ def setUp(self):
             self.activate_command = ''
             self.deactivate_command = ''
 
+    def inform_status(self):
+        encoding.debugging('VirtualEnv.inform_status')
+        if self.installed is False:
+            inform(u'Not found: "{}"'.format(self.activate_filename))
+
     @property
     def installed(self):
-        status = os.path.isfile(self.activate_filename)
-        if status is False:
-            inform(u'Missing virtualenv: "{}"'.format(self.activate_filename))
-        return status
+        encoding.debugging('VirtualEnv.installed')
+        return os.path.isfile(self.activate_filename)
 
-    def install_venv(self):
+    def install_venv(self, recreate, proxy_info):
+        encoding.debugging('VirtualEnv.install_venv')
         if self.path is not None:
-            if not self.installed:
+            if not self.installed or recreate:
                 commands = []
-                commands.extend(self.proxy.register_commands)
+                commands.extend(proxy_info.register_commands)
+                commands.append(
+                    'python -m pip install {} -U pip'.format(
+                        proxy_info.parameter)
+                    )
                 commands.append(
                     'python -m pip install {} --upgrade pip'.format(
-                        self.proxy.parameter)
+                        proxy_info.parameter)
                     )
                 commands.append(
-                    'pip install {} virtualenv'.format(self.proxy.parameter)
+                    'pip install {} virtualenv'.format(proxy_info.parameter)
                     )
                 commands.append(u'virtualenv "{}"'.format(self.path))
                 for cmd in commands:
-                    if 'teste' in self.proxy.parameter:
+                    if 'teste' in proxy_info.parameter:
                         encoding.display_message('Executaria\n  {}'.format(cmd))
                     else:
                         system.run_command(cmd)
@@ -120,55 +135,26 @@ def install_venv(self):
                     inform(u'Unable to create the virtualenv: "{}"'.format(self.path))
                     inform('Install the programs in a path which does not have diacritics')
 
-    def install_requirements(self):
-        commands = self.requirements.install_commands(
-                self.proxy,
-                uninstall=True)
-        self.execute_in_virtualenv(commands)
-
-    def execute_in_virtualenv(self, commands):
-        self.install_venv()
-        if self.installed:
-            _commands = [self.activate_command]
-            _commands.extend(commands)
-            self._execute_inline(_commands)
-
-    def _execute_inline(self, commands):
-        _commands = [item for item in commands if len(item) > 0]
-        cmd = self.sep.join(_commands)
-        display_cmd = self.proxy.hide_password(cmd)
-        self.logger.info(display_cmd)
-        if 'teste' in cmd:
-            encoding.display_message('Executaria\n  {}'.format(cmd))
-        else:
-            encoding.display_message(display_cmd)
-            system.run_command(cmd, False)
-
-    def activate(self):
-        if self.installed:
-            system.run_command(self.activate_command, True)
-
-    def deactivate(self):
-        if self.installed:
-            system.run_command(self.deactivate_command, True)
-
 
 class Requirements(object):
 
     def __init__(self, requirements_file):
         self.requirements_file = requirements_file
 
-    def install_commands(self, proxy, uninstall):
+    def install_commands(self, proxy_info, uninstall):
         commands = []
         if uninstall is True:
             commands = self.uninstall_commands()
-        commands.extend(proxy.register_commands)
+        commands.append(
+                    'python -m pip install {} -U pip'.format(
+                        proxy_info.parameter)
+                    )
         commands.append(
                     'python -m pip install {} --upgrade pip'.format(
-                        proxy.parameter)
+                        proxy_info.parameter)
                     )
         commands.append(u'pip install {} -r "{}"'.format(
-            proxy.parameter, self.requirements_file))
+            proxy_info.parameter, self.requirements_file))
         commands.append('pip freeze > python_libraries_installed.txt')
         return commands
 
@@ -181,28 +167,70 @@ def uninstall_commands(self):
 
 class AppCaller(object):
 
-    def __init__(self, logger, venv_path, req_file):
-        self.venv = VirtualEnv(logger, venv_path, req_file)
-
-    @property
-    def proxy(self):
-        return self.venv.proxy
-
-    @proxy.setter
-    def proxy(self, value):
-        self.venv.proxy = ProxyInfo(value)
-
-    def install_virtualenv(self, recreate=False):
-        inform('Install virtualenv')
-        self.venv.install_venv()
-        if self.venv.installed:
-            inform('Install virtualenv: done!')
-
-    def install_requirements(self):
-        if self.venv.installed:
-            inform('Install Requirements')
-            self.venv.install_requirements()
-        #inform('Install Requirements: done!')
+    def __init__(self, logger, venv_path, req_file, proxy_server_port):
+        encoding.debugging('AppCaller.AppCaller()')
+        self.logger = logger
+        self.venv = VirtualEnv(logger, venv_path)
+        self.requirements = Requirements(req_file)
+        self.proxy_server_port = proxy_server_port
+        self.proxy_info = ProxyInfo(None)
+
+    def install_requirements(self, restart, requirements_checker):
+        encoding.debugging(
+            'AppCaller.install_requirements',
+            (restart, requirements_checker))
+        reqs = 0
+        if requirements_checker is not None:
+            reqs = requirements_checker()
+        encoding.debugging(
+            'AppCaller.install_requirements',
+            (reqs, restart))
+        if restart or reqs > 0:
+            self.proxy_info = gen_proxy_info(self.proxy_server_port)
+            if restart or not self.venv.installed:
+                encoding.debugging(
+                    'AppCaller.install_requirements: self.venv.install_venv')
+                self.venv.install_venv(restart, self.proxy_info)
+            encoding.debugging(
+                'AppCaller.install_requirements: self.requirements.install_commands')
+            commands = self.requirements.install_commands(
+                self.proxy_info,
+                uninstall=restart)
+
+            self.execute(commands)
+
+    def format_commands(self, commands):
+        encoding.debugging('AppCaller.format_commands', commands)
+        _commands = []
+        for cmd in commands:
+            _cmd = []
+            if self.venv.installed:
+                _cmd = [self.venv.activate_command]
+            if self.proxy_info.parameter in cmd:
+                _cmd.extend(self.proxy_info.register_commands)
+            _cmd.append(cmd)
+            if self.venv.installed:
+                # _cmd.append(self.venv.deactivate_command)
+                _cmd = [self.commands_inline(_cmd)]
+            _commands.extend(_cmd)
+        encoding.debugging('AppCaller.format_commands: formatted', _commands)
+        return _commands
+
+    def commands_inline(self, commands):
+        encoding.debugging('AppCaller.commands_inline', commands)
+        _commands = [item for item in commands if len(item) > 0]
+        _cmd = self.venv.sep.join(_commands)
+        return _cmd
 
     def execute(self, commands):
-        self.venv.execute_in_virtualenv(commands)
+        encoding.debugging(
+            'AppCaller.execute',
+            commands)
+        for cmd in self.format_commands(commands):
+            encoding.debugging(
+                'AppCaller.execute',
+                cmd)
+            display_cmd = self.proxy_info.hide_password(cmd)
+            self.logger.info(display_cmd)
+            encoding.display_message(display_cmd)
+            os.system(cmd)

src/scielo/bin/xml/app_modules/app/config/config.py

@@ -454,5 +454,9 @@ def coded_table_required(self):
     def BLOCK_DISAGREEMENT_WITH_COLLECTION_CRITERIA(self):
         return self.is_activated('BLOCK_DISAGREEMENT_WITH_COLLECTION_CRITERIA')
 
+    @property
+    def USE_VIRTUAL_ENV(self):
+        return self.is_activated('USE_VIRTUAL_ENV')
+
     def is_activated(self, label, default='TRUE'):
         return self._data.get(label, default).upper() in ['ON', 'TRUE', 'YES']

src/scielo/bin/xml/app_modules/app/pkg_processors/pkg_processors.py

@@ -386,8 +386,16 @@ def normalized_package(self, xml_list):
         return package.Package(pkgfiles, outputs, workarea_path)
 
     def evaluate_package(self, pkg):
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - begin')
         registered_issue_data = registered.RegisteredIssue()
+
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - get_registered_issue_data')
         self.registered_issues_manager.get_registered_issue_data(pkg.issue_data, registered_issue_data)
+
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - affiliations')
         for xml_name in sorted(pkg.articles.keys()):
             a = pkg.articles[xml_name]
             if a is not None:
@@ -397,12 +405,28 @@ def evaluate_package(self, pkg):
                         institutions_results[aff_xml.id] = self.aff_normalizer.query_institutions(aff_xml)
                 pkg.articles[xml_name].institutions_query_results = institutions_results
                 pkg.articles[xml_name].normalized_affiliations = {aff_id: info[0] for aff_id, info in institutions_results.items()}
+
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - validate_pkg_articles')
         pkg_validations = self.validate_pkg_articles(pkg, registered_issue_data)
 
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - validate_merged_articles')
         articles_mergence = self.validate_merged_articles(pkg, registered_issue_data)
+
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - PkgArticlesValidationsReports')
         pkg_reports = pkg_articles_validations.PkgArticlesValidationsReports(pkg_validations, registered_issue_data.articles_db_manager is not None)
+
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - MergedArticlesReports')
         mergence_reports = merged_articles_validations.MergedArticlesReports(articles_mergence, registered_issue_data)
+
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - IssueArticlesValidationsReports')
         validations_reports = merged_articles_validations.IssueArticlesValidationsReports(pkg_reports, mergence_reports, self.is_xml_generation)
+        encoding.display_message(_('Evaluating package... Wait. '))
+        encoding.debugging('pkg_processors.make_package.evaluate_package - end')
         return registered_issue_data, validations_reports
 
     def make_package(self, pkg, GENERATE_PMC=False):
@@ -447,6 +471,7 @@ def validate_merged_articles(self, pkg, registered_issue_data):
             pkg.articles, self.is_db_generation)
 
     def report_result(self, pkg, validations_reports, conversion=None):
+        encoding.debugging('pkg_processors.make_package.report_result - begin')
         files_location = workarea.AssetsDestinations(pkg.wk.scielo_package_path, pkg.issue_data.acron, pkg.issue_data.issue_label)
         if conversion is not None:
             files_location = workarea.AssetsDestinations(pkg.wk.scielo_package_path, pkg.issue_data.acron, pkg.issue_data.issue_label, self.config.serial_path, self.config.local_web_app_path, self.config.web_app_site)
@@ -460,6 +485,7 @@ def report_result(self, pkg, validations_reports, conversion=None):
             for article_files in pkg.package_folder.pkgfiles_items.values():
                 # copia os xml para report path
                 article_files.copy_xml(reports.files_location.report_path)
+        encoding.debugging('pkg_processors.make_package.report_result - end')
         return reports
 
     def make_pmc_package(self, pkg, GENERATE_PMC):

src/scielo/bin/xml/app_modules/app/validations/article_content_validations.py

@@ -293,9 +293,11 @@ def article_type(self):
                             ' ; '.join(invalid)))
                 titles = [t.title for t in self.article.titles]
                 _titles = ' / '.join([u'"{}"'.format(t) for t in titles])
-                if utils.is_similar(self.article.toc_section, titles):
+                similar = utils.is_similar(self.article.toc_section, titles)
+                similar_alt = utils.is_similar_alternative(self.article.toc_section, titles)
+                if similar and similar_alt:
                     errors.append(
-                        _(u'{} must not be similar to the table of contents section "{}" '.format(
+                        _(u'"{}" must not be similar to the table of contents section "{}" '.format(
                             _titles, self.article.toc_section)))
         else:
             errors.append(