Implement load/save abilities for transformation rules (based on JSON file)

schema_transformer_helper.py

+# -*- coding: utf-8 -*-
+"""
+/***************************************************************************
+ SmallETL
+                                 A QGIS plugin
+ Simple ETL for spatial data
+ Generated by Plugin Builder: http://g-sherman.github.io/Qgis-Plugin-Builder/
+                              -------------------
+        begin                : 2022-07-07
+        git sha              : $Format:%H$
+        copyright            : (C) 2022 by OR2C
+        email                : or2c@univ-nantes.fr
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+"""
+from qgis.PyQt.QtCore import QObject, pyqtSignal, pyqtSlot
+from qgis.PyQt.QtWidgets import QFileDialog
+from qgis.core import Qgis, QgsMessageLog, QgsFeature, QgsExpression
+
+import json
+
+class SchemaTransformerHelper(QObject):
+    def __init__(self):
+        super(SchemaTransformerHelper, self).__init__()
+        
+    def save_rules_to_file(self, transform_definition: list) -> None:
+        '''
+        Save transformation rules to JSON file
+        '''
+        file_dialog_window = QFileDialog()                                              
+        file_dialog_window.setAcceptMode(1) #Means save File
+        file_dialog_window.setDefaultSuffix("json")
+        file_dialog_window.setNameFilters(["JSON files (*.json)"])
+        if file_dialog_window.exec_() == 0:
+            return
+        filename = file_dialog_window.selectedFiles()[0]
+        if filename:
+            with open(filename,'w') as file:
+                file_content = {}
+                file_content['rules'] = {}
+                for rule in transform_definition:
+                    field_number = rule[0]
+                    expression = rule[1].dump() # Returns an expression string because QgsExpression is not JSON serializable
+                    file_content['rules'][field_number] = expression
+                json.dump(file_content, file, indent = 4)
+
+    def load_rules_from_file(self) -> list:
+        '''
+        Load transformation rules from input JSON file.
+        Return list of tuples (field_number: int, expression: str)
+        '''
+        transformation_rules = []
+        file_dialog_window = QFileDialog()                                              
+        file_dialog_window.setAcceptMode(0) #Means read File
+        file_dialog_window.setDefaultSuffix("json")
+        file_dialog_window.setNameFilters(["JSON files (*.json)"])
+        if file_dialog_window.exec_() == 0:
+            return
+        filename = file_dialog_window.selectedFiles()[0]
+        with open(filename, 'r') as json_file:
+            data = json.load(json_file)
+            rules = data['rules']
+            for field_number, rule in rules.items():
+                rule_definition = (int(field_number), rule) # Return expression rule as String instead of QgsExpression
+                transformation_rules.append(rule_definition)
+        return transformation_rules

small_etl.py

@@ -35,6 +35,9 @@ from .resources import *
 from .small_etl_dialog import SmallETLDialog
 # Import worker
 from .small_etl_worker import ETLWorker
+# Import helper
+from .schema_transformer_helper import SchemaTransformerHelper
+
 import os.path
 
 
@@ -77,6 +80,9 @@ class SmallETL:
         self.worker = ETLWorker()
         self.thread = QThread()
 
+        # Initialize transformer helper
+        self.transformer_helper = SchemaTransformerHelper()
+
     # noinspection PyMethodMayBeStatic
     def tr(self, message):
         """Get the translation for a string using Qt translation API.
@@ -202,6 +208,23 @@ class SmallETL:
     def on_pb_cancel(self):
         self.thread.exit()
 
+    def on_pb_save(self):
+        transformation_def = self.generate_transformation_definition()
+        self.transformer_helper.save_rules_to_file(transformation_def)
+    
+    def on_pb_load(self):
+        '''
+        Load transformation rules from JSON file using helper.
+        Populate table widgets with expressions
+        '''
+        transformation_rules = self.transformer_helper.load_rules_from_file()
+        try:
+            for field_number, rule in transformation_rules:
+                exp_widget = self.dlg.tableWidget_targetFields.cellWidget(field_number, 1)
+                exp_widget.setExpression(rule) # setExpression expects a QString object and not QgsExpression
+        except Exception as err:
+            print(err)
+
     def on_progress(self, i):
         self.dlg.progressBar.setValue(i)
 
@@ -324,6 +347,9 @@ class SmallETL:
         # Connect change event for source and target layers comboboxes
         self.dlg.sourceMapLayerComboBox.currentIndexChanged.connect(self.populate_layer_tables)
         self.dlg.targetMapLayerComboBox.currentIndexChanged.connect(self.populate_layer_tables)
+        # Connect click event for Save and Load definition buttons
+        self.dlg.pushButton_saveRules.clicked.connect(self.on_pb_save)
+        self.dlg.pushButton_loadRules.clicked.connect(self.on_pb_load)
 
         # Show the dialog
         self.dlg.show()

small_etl_dialog_base.ui

@@ -252,7 +252,7 @@
    <property name="title">
     <string>Save Definition</string>
    </property>
-   <widget class="QPushButton" name="pushButton_loadTrafo">
+   <widget class="QPushButton" name="pushButton_loadRules">
     <property name="geometry">
      <rect>
       <x>110</x>
@@ -265,7 +265,7 @@
      <string>Load</string>
     </property>
    </widget>
-   <widget class="QPushButton" name="pushButton_saveTrafo">
+   <widget class="QPushButton" name="pushButton_saveRules">
     <property name="geometry">
      <rect>
       <x>10</x>

small_etl_worker.py

@@ -21,15 +21,9 @@
  *                                                                         *
  ***************************************************************************/
 """
-from operator import ne
 from qgis.PyQt.QtCore import QObject, pyqtSignal, pyqtSlot
-
 from qgis.core import Qgis, QgsMessageLog, QgsFeature, QgsExpressionContext, QgsExpressionContextUtils
 
-# Initialize Qt resources from file resources.py
-from .resources import *
-import time
-
 class ETLWorker(QObject):
     """
     Class to manage main ETL pipeline