diff --git a/modelbaker/iliwrapper/ili2dbconfig.py b/modelbaker/iliwrapper/ili2dbconfig.py index 487d832f..e03b18d4 100644 --- a/modelbaker/iliwrapper/ili2dbconfig.py +++ b/modelbaker/iliwrapper/ili2dbconfig.py @@ -587,3 +587,33 @@ def to_ili2db_args(self, extra_args=[], with_action=True): self.append_args(args, Ili2DbCommandConfiguration.to_ili2db_args(self)) return args + + +class Ili2CCommandConfiguration: + def __init__(self, other=None): + if not isinstance(other, Ili2CCommandConfiguration): + self.base_configuration = BaseConfiguration() + + self.oIMD16 = True + self.imdfile = "" + self.ilifile = "" + else: + # We got an 'other' object from which we'll get parameters + self.__dict__ = other.__dict__.copy() + + def append_args(self, args, values): + args += values + + def to_ili2c_args(self): + + args = self.base_configuration.to_ili2db_args(False, False) + + if self.oIMD16: + self.append_args(args, ["-oIMD16"]) + + if self.imdfile: + self.append_args(args, ["--out", self.imdfile]) + + if self.ilifile: + self.append_args(args, [self.ilifile]) + return args diff --git a/modelbaker/iliwrapper/ili2dbtools.py b/modelbaker/iliwrapper/ili2dbtools.py index 2d247545..22b52156 100644 --- a/modelbaker/iliwrapper/ili2dbtools.py +++ b/modelbaker/iliwrapper/ili2dbtools.py @@ -55,3 +55,13 @@ def get_tool_url(tool, db_ili_version): ) return "" + + +def get_ili2c_tool_version(): + return "5.6.6" + + +def get_ili2c_tool_url(): + return "https://downloads.interlis.ch/ili2c/ili2c-{version}.zip".format( + version=get_ili2c_tool_version() + ) diff --git a/modelbaker/iliwrapper/ili2dbutils.py b/modelbaker/iliwrapper/ili2dbutils.py index de58d724..1900882e 100644 --- a/modelbaker/iliwrapper/ili2dbutils.py +++ b/modelbaker/iliwrapper/ili2dbutils.py @@ -28,7 +28,12 @@ from ..utils.qt_utils import NetworkError, download_file from .globals import DbIliMode -from .ili2dbtools import get_tool_url, get_tool_version +from .ili2dbtools import ( + get_ili2c_tool_url, + get_ili2c_tool_version, + get_tool_url, + get_tool_version, +) def get_ili2db_bin(tool, db_ili_version, stdout, stderr): @@ -125,6 +130,75 @@ def get_ili2db_bin(tool, db_ili_version, stdout, stderr): return ili2db_file +def get_ili2c_bin(stdout, stderr): + ili_tool_version = get_ili2c_tool_version() + ili_tool_url = get_ili2c_tool_url() + + dir_path = os.path.dirname(os.path.realpath(__file__)) + ili2c_dir = "ili2c-{}".format(ili_tool_version) + + ili2c_file = os.path.join( + dir_path, + "bin", + ili2c_dir, + "ili2c.jar".format(version=ili_tool_version), + ) + + if not os.path.isfile(ili2c_file): + try: + os.makedirs(os.path.join(dir_path, "bin", ili2c_dir), exist_ok=True) + except FileExistsError: + pass + + tmpfile = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + + stdout.emit( + QCoreApplication.translate( + "ili2dbutils", + "Downloading ili2c version {}…".format(ili_tool_version), + ) + ) + + try: + download_file( + ili_tool_url, + tmpfile.name, + on_progress=lambda received, total: stdout.emit("."), + ) + except NetworkError as e: + stderr.emit( + QCoreApplication.translate( + "ili2dbutils", + 'Could not download ili2c\n\n Error: {error}\n\nFile "{file}" not found. Please download and extract ili2c'.format( + ili2db_url=ili_tool_url, + error=e.msg, + file=ili2c_file, + ), + ) + ) + return None + + try: + with zipfile.ZipFile(tmpfile.name, "r") as z: + z.extractall(os.path.join(dir_path, "bin", ili2c_dir)) + except zipfile.BadZipFile: + # We will realize soon enough that the files were not extracted + pass + + if not os.path.isfile(ili2c_file): + stderr.emit( + QCoreApplication.translate( + "ili2dbutils", + 'File "{file}" not found. Please download and extract ili2c.'.format( + file=ili2c_file, ili2c_url=ili_tool_url + ), + ) + ) + return None + + return ili2c_file + + def get_all_modeldir_in_path(path, lambdafunction=None): all_subdirs = [path[0] for path in os.walk(path)] # include path # Make sure path is included, it can be a special string like `%XTF_DIR` diff --git a/modelbaker/iliwrapper/ilicache.py b/modelbaker/iliwrapper/ilicache.py index 37856a15..028f803b 100644 --- a/modelbaker/iliwrapper/ilicache.py +++ b/modelbaker/iliwrapper/ilicache.py @@ -245,6 +245,9 @@ def _process_informationfile(self, file, netloc, url): model["version"] = self.get_element_text( model_metadata.find("ili23:Version", self.ns) ) + model["file"] = self.get_element_text( + model_metadata.find("ili23:File", self.ns) + ) model["repository"] = netloc repo_models.append(model) @@ -262,6 +265,9 @@ def _process_informationfile(self, file, netloc, url): model["version"] = self.get_element_text( model_metadata.find("ili23:Version", self.ns) ) + model["file"] = self.get_element_text( + model_metadata.find("ili23:File", self.ns) + ) model["repository"] = netloc repo_models.append(model) @@ -375,6 +381,7 @@ class IliModelItemModel(QStandardItemModel): class Roles(Enum): ILIREPO = Qt.ItemDataRole.UserRole + 1 VERSION = Qt.ItemDataRole.UserRole + 2 + FILE = Qt.ItemDataRole.UserRole + 3 def __int__(self): return self.value @@ -400,6 +407,7 @@ def set_repositories(self, repositories): ) # considered in completer item.setData(model["repository"], int(IliModelItemModel.Roles.ILIREPO)) item.setData(model["version"], int(IliModelItemModel.Roles.VERSION)) + item.setData(model["file"], int(IliModelItemModel.Roles.FILE)) names.append(model["name"]) self.appendRow(item) diff --git a/modelbaker/iliwrapper/ilicompiler.py b/modelbaker/iliwrapper/ilicompiler.py new file mode 100644 index 00000000..0a11bbd7 --- /dev/null +++ b/modelbaker/iliwrapper/ilicompiler.py @@ -0,0 +1,45 @@ +""" +/*************************************************************************** + ------------------- + begin : 25/11/11 + git sha : :%H$ + copyright : (C) 2017 by OPENGIS.ch + email : info@opengis.ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" +from .ili2dbconfig import Ili2CCommandConfiguration +from .ili2dbutils import get_ili2c_bin +from .iliexecutable import IliExecutable + + +class IliCompiler(IliExecutable): + def __init__(self, parent=None): + super().__init__(parent) + + def _create_config(self) -> Ili2CCommandConfiguration: + """Creates the configuration that will be used by *run* method. + :return: ili2c configuration""" + return Ili2CCommandConfiguration() + + def _args(self, param): + """Gets the list of ili2c arguments from configuration. + :return: ili2c arguments list. + :rtype: list + """ + # todo care about param (it should not be considered) + return self.configuration.to_ili2c_args() + + def _ili2_jar_arg(self): + ili2c_bin = get_ili2c_bin(self.stdout, self.stderr) + if not ili2c_bin: + return self.ILI2C_NOT_FOUND + return ["-jar", ili2c_bin] diff --git a/modelbaker/iliwrapper/iliexecutable.py b/modelbaker/iliwrapper/iliexecutable.py index 34f61804..a6514c65 100644 --- a/modelbaker/iliwrapper/iliexecutable.py +++ b/modelbaker/iliwrapper/iliexecutable.py @@ -79,7 +79,7 @@ def _args(self, hide_password): return get_ili2db_args(self.configuration, hide_password) - def _ili2db_jar_arg(self): + def _ili2_jar_arg(self): ili2db_bin = get_ili2db_bin( self.tool, self._get_ili2db_version(), self.stdout, self.stderr ) @@ -95,7 +95,7 @@ def _escaped_arg(self, argument): return argument def command(self, hide_password): - ili2db_jar_arg = self._ili2db_jar_arg() + ili2db_jar_arg = self._ili2_jar_arg() if ili2db_jar_arg == self.ILI2DB_NOT_FOUND: return "ili2db tool not found!" @@ -142,7 +142,7 @@ def run(self, edited_command=None): ) if not edited_command: - ili2db_jar_arg = self._ili2db_jar_arg() + ili2db_jar_arg = self._ili2_jar_arg() if ili2db_jar_arg == self.ILI2DB_NOT_FOUND: return self.ILI2DB_NOT_FOUND args = self._args(False) diff --git a/modelbaker/pythonizer/pythonizer.py b/modelbaker/pythonizer/pythonizer.py new file mode 100644 index 00000000..3482ff23 --- /dev/null +++ b/modelbaker/pythonizer/pythonizer.py @@ -0,0 +1,135 @@ +import datetime +import os + +from ili2py.mappers.helpers import Index +from ili2py.readers.interlis_24.ilismeta16.xsdata import Imd16Reader +from ili2py.writers.py.python_structure import Library +from qgis.core import Qgis +from qgis.PyQt.QtCore import QFile, QObject, QStandardPaths + +from ..iliwrapper.ili2dbconfig import Ili2CCommandConfiguration +from ..iliwrapper.ili2dbutils import JavaNotFoundError +from ..iliwrapper.ilicompiler import IliCompiler +from ..utils import db_utils +from ..utils.globals import default_log_function +from ..utils.qt_utils import NetworkError, download_file + + +class Pythonizer(QObject): + """ + This is pure Tinkerlis. pythonizer function does the ili2py stuff. The rest is kind of a utils api. + """ + + def __init__(self, log_function=None) -> None: + QObject.__init__(self) + + self.log_function = log_function if log_function else default_log_function + + if not log_function: + self.log_function = default_log_function + + def pythonize(self, imd_file): + reader = Imd16Reader() + metamodel = reader.read(imd_file) + index = Index(metamodel.datasection) + library_name = index.types_bucket["Model"][-1].name + library = Library.from_imd(metamodel.datasection.ModelData, index, library_name) + return index, library + + def compile(self, base_configuration, ili_file): + compiler = IliCompiler() + + configuration = Ili2CCommandConfiguration() + configuration.base_configuration = base_configuration + configuration.ilifile = ili_file + configuration.imdfile = os.path.join( + QStandardPaths.writableLocation( + QStandardPaths.StandardLocation.TempLocation + ), + "temp_imd_{:%Y%m%d%H%M%S%f}.imd".format(datetime.datetime.now()), + ) + + compiler.configuration = configuration + + compiler.stdout.connect(self.on_ili_stdout) + compiler.stderr.connect(self.on_ili_stderr) + compiler.process_started.connect(self.on_ili_process_started) + compiler.process_finished.connect(self.on_ili_process_finished) + result = True + + try: + compiler_result = compiler.run() + if compiler_result != compiler.SUCCESS: + result = False + except JavaNotFoundError as e: + self.log_function( + self.tr("Java not found error: {}").format(e.error_string), + Qgis.MessageLevel.Warning, + ) + result = False + + return result, compiler.configuration.imdfile + + def model_files_generated_from_db(self, configuration, model_list=[]): + model_files = [] + # this could be improved i guess, we already have the models read from the same function. but yes. poc etc. + db_connector = db_utils.get_db_connector(configuration) + db_connector.get_models() + model_records = db_connector.get_models() + for record in model_records: + name = record["modelname"].split("{")[0] + # on an empty model_list we create a file for every found model + if len(model_list) == 0 or name in model_list: + modelfilepath = self._temp_ilifile(name) + file = QFile(modelfilepath) + if file.open(QFile.OpenModeFlag.WriteOnly): + file.write(record["content"].encode("utf-8")) + file.close() + model_files.append(modelfilepath) + print(modelfilepath) + return model_files + + def download_file(self, modelname, url): + modelfilepath = self._temp_ilifile(modelname) + try: + download_file( + url, + modelfilepath, + on_progress=lambda received, total: self.on_ili_stdout("."), + ) + except NetworkError: + self.on_ili_stderr(f"Could not download model {modelname} from {url}") + return None + return modelfilepath + + def _temp_ilifile(self, name): + return os.path.join( + QStandardPaths.writableLocation( + QStandardPaths.StandardLocation.TempLocation + ), + "temp_{}_{:%Y%m%d%H%M%S%f}.ili".format(name, datetime.datetime.now()), + ) + + def on_ili_stdout(self, message): + lines = message.strip().split("\n") + for line in lines: + text = f"pythonizer: {line}" + self.log_function(text, Qgis.MessageLevel.Info) + + def on_ili_stderr(self, message): + lines = message.strip().split("\n") + for line in lines: + text = f"pythonizer: {line}" + self.log_function(text, Qgis.MessageLevel.Critical) + + def on_ili_process_started(self, command): + text = f"pythonizer: {command}" + self.log_function(text, Qgis.MessageLevel.Info) + + def on_ili_process_finished(self, exit_code, result): + if exit_code == 0: + text = f"pythonizer: Successfully performed command." + self.log_function(text, Qgis.MessageLevel.Info) + else: + text = f"pythonizer: Finished with errors: {result}" + self.log_function(text, Qgis.MessageLevel.Critical) diff --git a/modelbaker/pythonizer/settings_prophet.py b/modelbaker/pythonizer/settings_prophet.py new file mode 100644 index 00000000..3250d614 --- /dev/null +++ b/modelbaker/pythonizer/settings_prophet.py @@ -0,0 +1,154 @@ +from ili2py.mappers.helpers import Index +from qgis.PyQt.QtCore import QFile, QObject + +from ..utils.globals import default_log_function + + +class SettingsProphet(QObject): + def __init__(self, index: Index, models: list, log_function=None) -> None: + QObject.__init__(self) + + self.log_function = log_function if log_function else default_log_function + + if not log_function: + self.log_function = default_log_function + + self.index = index + self.models = models + + def smart_inheritance(self): + """ + Does it make sense to make any suggestions here? I don't know. + """ + return True + + def enum_info(self): + + return True + + def has_basket_oids(self): + """ + Is there any BASKET OID definition in the model. + """ + bid_in_model = self.index.basket_oid_in_model + bid_in_topics = self.index.basket_oid_in_submodel + if len(bid_in_model.keys()) + len(bid_in_topics.keys()): + return True + return False + + def has_arcs(self): + """ + Arcs in any classes of the model or imported models. + """ + # get all the geometric attributes of the relevant classes + relevant_geometric_attributes = self._relevant_geometric_attributes() + + # get the line form of the relevant geometry attributes + line_forms = self.index.geometric_attributes_line_form + line_forms_of_interest = [] + for attribute in line_forms.keys(): + if attribute in relevant_geometric_attributes: + line_forms_of_interest += line_forms[attribute] + + return bool( + "INTERLIS.ARCS" in line_forms_of_interest + or "ARCS" in line_forms_of_interest + ) + + def has_multiple_geometry_columns(self): + """ + Multiple geometry columns in any classes of the model or imported models. + """ + relevant_geometric_attributes_per_class = ( + self._relevant_geometric_attributes_per_class() + ) + if any( + len(columns) > 1 + for columns in relevant_geometric_attributes_per_class.values() + ): + return True + return False + + def multi_geometry_structures_on_23(self) -> dict: + """ + Multi geometry structures in INTERLIS 23.. + """ + set_of_relevant_geometric_attributes = set( + self._relevant_geometric_attributes() + ) + set_of_multi_geometry_attributes = set(self.index.geometric_attributes_multi) + + relevant_multi_geometry_attributes = list( + set_of_relevant_geometric_attributes & set_of_multi_geometry_attributes + ) + + # then it should get the type of the attributes (like KbS_V1_5.Belastete_Standorte.MultiPolygon) + # and then it should return KbS_V1_5.Belastete_Standorte.MultiPolygon : 'polygon' or 'surface' or something... + return {} + + def _relevant_classes(self): + # get the relevant baskets of the models + relevant_topics = [] + all_topics = self.index.submodel_in_package + for model in self.models: + if model in all_topics.keys(): + relevant_topics += all_topics[model] + topic_baskets_map = self.index.topic_basket + relevant_baskets = [topic_baskets_map.get(topic) for topic in relevant_topics] + + # get all the relevant classes by checking if they are allowed in the data unit + relevant_classes = [] + all_elements = self.index.allowed_in_basket_of_data_unit + for element_basket in all_elements.keys(): + if element_basket in relevant_baskets: + relevant_classes += all_elements[element_basket] + return relevant_classes + + def _relevant_geometric_attributes_per_class(self) -> dict: + relevant_classes = self._relevant_classes() + geometric_classes = self.index.geometric_classes + relevant_geometric_attributes_per_class = {} + for geometric_classname in geometric_classes.keys(): + if geometric_classname in relevant_classes: + relevant_geometric_attributes_per_class[ + geometric_classname + ] = geometric_classes[geometric_classname] + return relevant_geometric_attributes_per_class + + def _relevant_geometric_attributes(self) -> list: + relevant_geometric_attributes = [] + relevant_geometric_attributes_per_class = ( + self._relevant_geometric_attributes_per_class() + ) + for relevant_classname in relevant_geometric_attributes_per_class.keys(): + relevant_geometric_attributes += relevant_geometric_attributes_per_class[ + relevant_classname + ] + return relevant_geometric_attributes + + def is_translation(self): + return False + + +class ProphetTools(QObject): + def __init__(self, log_function=None) -> None: + QObject.__init__(self) + + self.log_function = log_function if log_function else default_log_function + + def _multi_geometry_mappings(self, multigeometry_structs): + entries = [] + for element in multigeometry_structs: + entries.append("[{}]\n{}".format(element, "test")) + return "\n".join(entries) + + def temp_meta_attr_file(self, multigeometry_structs): + temporary_filename = "{}/modelbaker-metaattrs.conf".format(QDir.tempPath()) + temporary_file = QFile(temporary_filename) + if temporary_file.open(QFile.OpenModeFlag.WriteOnly): + content = self._multi_geometry_mappings(multigeometry_structs) + if content: + temporary_file.write(content).encode("utf-8") + temporary_file.close() + return temporary_filename + return None diff --git a/tests/test_pythonizer.py b/tests/test_pythonizer.py new file mode 100644 index 00000000..ac488d48 --- /dev/null +++ b/tests/test_pythonizer.py @@ -0,0 +1,84 @@ +""" +/*************************************************************************** + begin : 08.12.2025 + git sha : :%H$ + copyright : (C) 2025 by Dave Signer + email : david at opengis ch + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +import tempfile + +from qgis.testing import start_app, unittest + +from modelbaker.iliwrapper.ili2dbconfig import BaseConfiguration +from modelbaker.pythonizer.pythonizer import Pythonizer +from modelbaker.pythonizer.settings_prophet import ProphetTools, SettingsProphet +from tests.utils import testdata_path + +start_app() + + +class TestPythonizer(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Run before all tests.""" + cls.basetestpath = tempfile.mkdtemp() + cls.base_config = BaseConfiguration() + + def test_settings_prophet_nupla(self): + pythonizer = Pythonizer() + base_configuration = BaseConfiguration() + ili_file = testdata_path("ilimodels/Nutzungsplanung_V1_2.ili") + john_wayne, imd_file = pythonizer.compile(base_configuration, ili_file) + index, lib = pythonizer.pythonize(imd_file) + prophet = SettingsProphet(index) + + assert prophet.has_basket_oids() + assert prophet.has_arcs() + assert not prophet.has_multiple_geometrie_columms() + + expected_multigeometries = {} + assert prophet.multi_geometry_structures_on_23() == expected_multigeometries + + def test_settings_prophet_arcmodel(self): + pythonizer = Pythonizer() + base_configuration = BaseConfiguration() + ili_file = testdata_path("ilimodels/KT_ArcInfrastruktur.ili") + john_wayne, imd_file = pythonizer.compile(base_configuration, ili_file) + index, lib = pythonizer.pythonize(imd_file) + prophet = SettingsProphet(index) + + assert not prophet.has_basket_oids() + assert prophet.has_arcs() + assert prophet.has_multiple_geometrie_columms() + + expected_multigeometries = {} + assert prophet.multi_geometry_structures_on_23() == expected_multigeometries + + def test_settings_prophet_arcs(self): + pythonizer = Pythonizer() + base_configuration = BaseConfiguration() + ili_file = testdata_path("ilimodels/KbS_V1_5.ili") + john_wayne, imd_file = pythonizer.compile(base_configuration, ili_file) + index, lib = pythonizer.pythonize(imd_file) + prophet = SettingsProphet(index) + + assert not prophet.has_basket_oids() + assert not prophet.has_arcs() + assert prophet.has_multiple_geometrie_columms() + + expected_multigeometries = {} + assert prophet.multi_geometry_structures_on_23() == expected_multigeometries + + prophet_tools = ProphetTools() + prophet_tools.temp_meta_attr_file(expected_multigeometries) diff --git a/tests/testdata/ilimodels/ArcInfrastruktur_V1.ili b/tests/testdata/ilimodels/ArcInfrastruktur_V1.ili new file mode 100644 index 00000000..1150b2c3 --- /dev/null +++ b/tests/testdata/ilimodels/ArcInfrastruktur_V1.ili @@ -0,0 +1,36 @@ +INTERLIS 2.3; + +MODEL ArcInfrastruktur_V1 (en) AT "https://modelbaker.ch" VERSION "2023-03-29" = + IMPORTS GeometryCHLV95_V1; + + DOMAIN + Line = POLYLINE WITH (STRAIGHTS) VERTEX GeometryCHLV95_V1.Coord2; + Surface = SURFACE WITH (STRAIGHTS) VERTEX GeometryCHLV95_V1.Coord2 WITHOUT OVERLAPS > 0.001; + LineArcs = POLYLINE WITH (STRAIGHTS, ARCS) VERTEX GeometryCHLV95_V1.Coord2; + SurfaceArcs = SURFACE WITH (STRAIGHTS, ARCS) VERTEX GeometryCHLV95_V1.Coord2 WITHOUT OVERLAPS > 0.001; + + TOPIC StrassenEtc = + CLASS Strasse = + Name : MANDATORY TEXT*99; + Geometrie : MANDATORY ArcInfrastruktur_V1.Line; + END Strasse; + + CLASS Haus = + Name : MANDATORY TEXT*99; + Geometrie : MANDATORY ArcInfrastruktur_V1.SurfaceArcs; + END Haus; + END StrassenEtc; + + TOPIC Natur = + CLASS Pfad = + Name : MANDATORY TEXT*99; + Geometrie : MANDATORY ArcInfrastruktur_V1.LineArcs; + END Pfad; + + CLASS Park = + Name : MANDATORY TEXT*99; + Geometrie : MANDATORY ArcInfrastruktur_V1.Surface; + END Park; + END Natur; + +END ArcInfrastruktur_V1. diff --git a/tests/testdata/ilimodels/KT_ArcInfrastruktur_V1.ili b/tests/testdata/ilimodels/KT_ArcInfrastruktur_V1.ili new file mode 100644 index 00000000..a5bde115 --- /dev/null +++ b/tests/testdata/ilimodels/KT_ArcInfrastruktur_V1.ili @@ -0,0 +1,19 @@ + +INTERLIS 2.3; + +/* Ortsplanung as national model */ +MODEL KT_ArcInfrastruktur_V1 (en) AT "https://modelbaker.ch" VERSION "2023-03-29" = + IMPORTS ArcInfrastruktur_V1; + + TOPIC StrassenEtc EXTENDS ArcInfrastruktur_V1.StrassenEtc = + CLASS Strasse (EXTENDED) = + Beschreibung : MANDATORY TEXT*99; + END Strasse; + + CLASS Konstruktion = + Name : TEXT; + Linie : MANDATORY ArcInfrastruktur_V1.Line; + Polygon : MANDATORY ArcInfrastruktur_V1.Surface; + END Konstruktion; + END StrassenEtc; +END KT_ArcInfrastruktur_V1.