PoC ili2py R&D Backend - Don't merge!

modelbaker/iliwrapper/ili2dbconfig.py

@@ -587,3 +587,33 @@ def to_ili2db_args(self, extra_args=[], with_action=True):
         self.append_args(args, Ili2DbCommandConfiguration.to_ili2db_args(self))
 
         return args
+
+
+class Ili2CCommandConfiguration:
+    def __init__(self, other=None):
+        if not isinstance(other, Ili2CCommandConfiguration):
+            self.base_configuration = BaseConfiguration()
+
+            self.oIMD16 = True
+            self.imdfile = ""
+            self.ilifile = ""
+        else:
+            # We got an 'other' object from which we'll get parameters
+            self.__dict__ = other.__dict__.copy()
+
+    def append_args(self, args, values):
+        args += values
+
+    def to_ili2c_args(self):
+
+        args = self.base_configuration.to_ili2db_args(False, False)
+
+        if self.oIMD16:
+            self.append_args(args, ["-oIMD16"])
+
+        if self.imdfile:
+            self.append_args(args, ["--out", self.imdfile])
+
+        if self.ilifile:
+            self.append_args(args, [self.ilifile])
+        return args

modelbaker/iliwrapper/ili2dbtools.py

@@ -55,3 +55,13 @@ def get_tool_url(tool, db_ili_version):
         )
 
     return ""
+
+
+def get_ili2c_tool_version():
+    return "5.6.6"
+
+
+def get_ili2c_tool_url():
+    return "https://downloads.interlis.ch/ili2c/ili2c-{version}.zip".format(
+        version=get_ili2c_tool_version()
+    )

modelbaker/iliwrapper/ili2dbutils.py

@@ -28,7 +28,12 @@
 
 from ..utils.qt_utils import NetworkError, download_file
 from .globals import DbIliMode
-from .ili2dbtools import get_tool_url, get_tool_version
+from .ili2dbtools import (
+    get_ili2c_tool_url,
+    get_ili2c_tool_version,
+    get_tool_url,
+    get_tool_version,
+)
 
 
 def get_ili2db_bin(tool, db_ili_version, stdout, stderr):
@@ -125,6 +130,75 @@ def get_ili2db_bin(tool, db_ili_version, stdout, stderr):
     return ili2db_file
 
 
+def get_ili2c_bin(stdout, stderr):
+    ili_tool_version = get_ili2c_tool_version()
+    ili_tool_url = get_ili2c_tool_url()
+
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    ili2c_dir = "ili2c-{}".format(ili_tool_version)
+
+    ili2c_file = os.path.join(
+        dir_path,
+        "bin",
+        ili2c_dir,
+        "ili2c.jar".format(version=ili_tool_version),
+    )
+
+    if not os.path.isfile(ili2c_file):
+        try:
+            os.makedirs(os.path.join(dir_path, "bin", ili2c_dir), exist_ok=True)
+        except FileExistsError:
+            pass
+
+        tmpfile = tempfile.NamedTemporaryFile(suffix=".zip", delete=False)
+
+        stdout.emit(
+            QCoreApplication.translate(
+                "ili2dbutils",
+                "Downloading ili2c version {}…".format(ili_tool_version),
+            )
+        )
+
+        try:
+            download_file(
+                ili_tool_url,
+                tmpfile.name,
+                on_progress=lambda received, total: stdout.emit("."),
+            )
+        except NetworkError as e:
+            stderr.emit(
+                QCoreApplication.translate(
+                    "ili2dbutils",
+                    'Could not download ili2c\n\n  Error: {error}\n\nFile "{file}" not found. Please download and extract <a href="{ili2db_url}">ili2c</a>'.format(
+                        ili2db_url=ili_tool_url,
+                        error=e.msg,
+                        file=ili2c_file,
+                    ),
+                )
+            )
+            return None
+
+        try:
+            with zipfile.ZipFile(tmpfile.name, "r") as z:
+                z.extractall(os.path.join(dir_path, "bin", ili2c_dir))
+        except zipfile.BadZipFile:
+            # We will realize soon enough that the files were not extracted
+            pass
+
+        if not os.path.isfile(ili2c_file):
+            stderr.emit(
+                QCoreApplication.translate(
+                    "ili2dbutils",
+                    'File "{file}" not found. Please download and extract <a href="{ili2c_url}">ili2c</a>.'.format(
+                        file=ili2c_file, ili2c_url=ili_tool_url
+                    ),
+                )
+            )
+            return None
+
+    return ili2c_file
+
+
 def get_all_modeldir_in_path(path, lambdafunction=None):
     all_subdirs = [path[0] for path in os.walk(path)]  # include path
     # Make sure path is included, it can be a special string like `%XTF_DIR`

modelbaker/iliwrapper/ilicache.py

@@ -245,6 +245,9 @@ def _process_informationfile(self, file, netloc, url):
                     model["version"] = self.get_element_text(
                         model_metadata.find("ili23:Version", self.ns)
                     )
+                    model["file"] = self.get_element_text(
+                        model_metadata.find("ili23:File", self.ns)
+                    )
                     model["repository"] = netloc
                     repo_models.append(model)
 
@@ -262,6 +265,9 @@ def _process_informationfile(self, file, netloc, url):
                     model["version"] = self.get_element_text(
                         model_metadata.find("ili23:Version", self.ns)
                     )
+                    model["file"] = self.get_element_text(
+                        model_metadata.find("ili23:File", self.ns)
+                    )
                     model["repository"] = netloc
                     repo_models.append(model)
 
@@ -375,6 +381,7 @@ class IliModelItemModel(QStandardItemModel):
     class Roles(Enum):
         ILIREPO = Qt.ItemDataRole.UserRole + 1
         VERSION = Qt.ItemDataRole.UserRole + 2
+        FILE = Qt.ItemDataRole.UserRole + 3
 
         def __int__(self):
             return self.value
@@ -400,6 +407,7 @@ def set_repositories(self, repositories):
                 )  # considered in completer
                 item.setData(model["repository"], int(IliModelItemModel.Roles.ILIREPO))
                 item.setData(model["version"], int(IliModelItemModel.Roles.VERSION))
+                item.setData(model["file"], int(IliModelItemModel.Roles.FILE))
 
                 names.append(model["name"])
                 self.appendRow(item)

modelbaker/iliwrapper/ilicompiler.py

@@ -0,0 +1,45 @@
+"""
+/***************************************************************************
+                              -------------------
+        begin                : 25/11/11
+        git sha              : :%H$
+        copyright            : (C) 2017 by OPENGIS.ch
+        email                : [email protected]
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+"""
+from .ili2dbconfig import Ili2CCommandConfiguration
+from .ili2dbutils import get_ili2c_bin
+from .iliexecutable import IliExecutable
+
+
+class IliCompiler(IliExecutable):
+    def __init__(self, parent=None):
+        super().__init__(parent)
+
+    def _create_config(self) -> Ili2CCommandConfiguration:
+        """Creates the configuration that will be used by *run* method.
+        :return: ili2c configuration"""
+        return Ili2CCommandConfiguration()
+
+    def _args(self, param):
+        """Gets the list of ili2c arguments from configuration.
+        :return: ili2c arguments list.
+        :rtype: list
+        """
+        # todo care about param (it should not be considered)
+        return self.configuration.to_ili2c_args()
+
+    def _ili2_jar_arg(self):
+        ili2c_bin = get_ili2c_bin(self.stdout, self.stderr)
+        if not ili2c_bin:
+            return self.ILI2C_NOT_FOUND
+        return ["-jar", ili2c_bin]

modelbaker/iliwrapper/iliexecutable.py

@@ -79,7 +79,7 @@ def _args(self, hide_password):
 
         return get_ili2db_args(self.configuration, hide_password)
 
-    def _ili2db_jar_arg(self):
+    def _ili2_jar_arg(self):
         ili2db_bin = get_ili2db_bin(
             self.tool, self._get_ili2db_version(), self.stdout, self.stderr
         )
@@ -95,7 +95,7 @@ def _escaped_arg(self, argument):
         return argument
 
     def command(self, hide_password):
-        ili2db_jar_arg = self._ili2db_jar_arg()
+        ili2db_jar_arg = self._ili2_jar_arg()
         if ili2db_jar_arg == self.ILI2DB_NOT_FOUND:
             return "ili2db tool not found!"
 
@@ -142,7 +142,7 @@ def run(self, edited_command=None):
         )
 
         if not edited_command:
-            ili2db_jar_arg = self._ili2db_jar_arg()
+            ili2db_jar_arg = self._ili2_jar_arg()
             if ili2db_jar_arg == self.ILI2DB_NOT_FOUND:
                 return self.ILI2DB_NOT_FOUND
             args = self._args(False)

modelbaker/pythonizer/pythonizer.py

@@ -0,0 +1,135 @@
+import datetime
+import os
+
+from ili2py.mappers.helpers import Index
+from ili2py.readers.interlis_24.ilismeta16.xsdata import Imd16Reader
+from ili2py.writers.py.python_structure import Library
+from qgis.core import Qgis
+from qgis.PyQt.QtCore import QFile, QObject, QStandardPaths
+
+from ..iliwrapper.ili2dbconfig import Ili2CCommandConfiguration
+from ..iliwrapper.ili2dbutils import JavaNotFoundError
+from ..iliwrapper.ilicompiler import IliCompiler
+from ..utils import db_utils
+from ..utils.globals import default_log_function
+from ..utils.qt_utils import NetworkError, download_file
+
+
+class Pythonizer(QObject):
+    """
+    This is pure Tinkerlis. pythonizer function does the ili2py stuff. The rest is kind of a utils api.
+    """
+
+    def __init__(self, log_function=None) -> None:
+        QObject.__init__(self)
+
+        self.log_function = log_function if log_function else default_log_function
+
+        if not log_function:
+            self.log_function = default_log_function
+
+    def pythonize(self, imd_file):
+        reader = Imd16Reader()
+        metamodel = reader.read(imd_file)
+        index = Index(metamodel.datasection)
+        library_name = index.types_bucket["Model"][-1].name
+        library = Library.from_imd(metamodel.datasection.ModelData, index, library_name)
+        return index, library
+
+    def compile(self, base_configuration, ili_file):
+        compiler = IliCompiler()
+
+        configuration = Ili2CCommandConfiguration()
+        configuration.base_configuration = base_configuration
+        configuration.ilifile = ili_file
+        configuration.imdfile = os.path.join(
+            QStandardPaths.writableLocation(
+                QStandardPaths.StandardLocation.TempLocation
+            ),
+            "temp_imd_{:%Y%m%d%H%M%S%f}.imd".format(datetime.datetime.now()),
+        )
+
+        compiler.configuration = configuration
+
+        compiler.stdout.connect(self.on_ili_stdout)
+        compiler.stderr.connect(self.on_ili_stderr)
+        compiler.process_started.connect(self.on_ili_process_started)
+        compiler.process_finished.connect(self.on_ili_process_finished)
+        result = True
+
+        try:
+            compiler_result = compiler.run()
+            if compiler_result != compiler.SUCCESS:
+                result = False
+        except JavaNotFoundError as e:
+            self.log_function(
+                self.tr("Java not found error: {}").format(e.error_string),
+                Qgis.MessageLevel.Warning,
+            )
+            result = False
+
+        return result, compiler.configuration.imdfile
+
+    def model_files_generated_from_db(self, configuration, model_list=[]):
+        model_files = []
+        # this could be improved i guess, we already have the models read from the same function. but yes. poc etc.
+        db_connector = db_utils.get_db_connector(configuration)
+        db_connector.get_models()
+        model_records = db_connector.get_models()
+        for record in model_records:
+            name = record["modelname"].split("{")[0]
+            # on an empty model_list we create a file for every found model
+            if len(model_list) == 0 or name in model_list:
+                modelfilepath = self._temp_ilifile(name)
+                file = QFile(modelfilepath)
+                if file.open(QFile.OpenModeFlag.WriteOnly):
+                    file.write(record["content"].encode("utf-8"))
+                    file.close()
+                model_files.append(modelfilepath)
+                print(modelfilepath)
+        return model_files
+
+    def download_file(self, modelname, url):
+        modelfilepath = self._temp_ilifile(modelname)
+        try:
+            download_file(
+                url,
+                modelfilepath,
+                on_progress=lambda received, total: self.on_ili_stdout("."),
+            )
+        except NetworkError:
+            self.on_ili_stderr(f"Could not download model {modelname} from {url}")
+            return None
+        return modelfilepath
+
+    def _temp_ilifile(self, name):
+        return os.path.join(
+            QStandardPaths.writableLocation(
+                QStandardPaths.StandardLocation.TempLocation
+            ),
+            "temp_{}_{:%Y%m%d%H%M%S%f}.ili".format(name, datetime.datetime.now()),
+        )
+
+    def on_ili_stdout(self, message):
+        lines = message.strip().split("\n")
+        for line in lines:
+            text = f"pythonizer: {line}"
+            self.log_function(text, Qgis.MessageLevel.Info)
+
+    def on_ili_stderr(self, message):
+        lines = message.strip().split("\n")
+        for line in lines:
+            text = f"pythonizer: {line}"
+            self.log_function(text, Qgis.MessageLevel.Critical)
+
+    def on_ili_process_started(self, command):
+        text = f"pythonizer: {command}"
+        self.log_function(text, Qgis.MessageLevel.Info)
+
+    def on_ili_process_finished(self, exit_code, result):
+        if exit_code == 0:
+            text = f"pythonizer: Successfully performed command."
+            self.log_function(text, Qgis.MessageLevel.Info)
+        else:
+            text = f"pythonizer: Finished with errors: {result}"
+            self.log_function(text, Qgis.MessageLevel.Critical)