debugged problems

app/config.py

@@ -39,6 +39,7 @@ class TestingConfig(Config):
 
 
 class LocalConfig(Config):
+    SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postgres:postgres@localhost:5432/dbdq_test'
     IO = 'http://localhost:5300'
     DEBUG = True
     AUTO_UPDATE = False

app/main/__init__.py

@@ -6,5 +6,5 @@ main = Blueprint('main', __name__)
 CORS(main)
 
 from app.main import errors
-from app.main.views import life_check, validate
+from app.main.views import life_check, validate, rulegroup, ruleset
 from app import db

app/main/utils/common.py

@@ -5,11 +5,11 @@ import numpy as np
 import pandas as pd
 from app.errors import *
 import gc
-from rulemall.utils.rule_builder import rule_builder
 from app import db
 from app.models import Validation
 import yaml
 import requests
+from app.main.utils.db_util import return_rg
 
 
 def response_message(data=None, status=200):
@@ -60,26 +60,50 @@ def convert(o):
     json.dumps(o)
 
 
-def read_data(ds=None, uid=None, cols=[]):
-    if ds:
-        req = requests.post(f'{current_app.config["IO"]}/r/data/{ds}/{uid}', json={'columns': cols})
+def read_data(puid=None, ruleset=None, cols=[], path=None):
+    if puid:
+        if ruleset:
+            ruleset.data['type'] = 'rp'
+            uid = ruleset.data['uid']
+        else:
+            uid = puid
+        if cols:
+            req = requests.post(f'{current_app.config["IO"]}/get_data/{uid}', json={'columns': cols})
+        else:
+            req = requests.post(f'{current_app.config["IO"]}/get_data/{uid}', json={'form_code': 'all'})
         return pd.DataFrame(req.json()['data'])
     else:
-        return pd.read_excel('app/main/static/sample_data.xlsx')
+        if not path:
+            path = 'app/main/static/sample_data.xlsx'
+        return pd.read_excel(path)
 
 
-def read_rules():
-    path = 'app/main/static/rule_config.yml'
-    return yaml.load(open(path), Loader=yaml.FullLoader)
+def read_rules(rsid=None, rgid=None, where='file', path=None):
+    if where == 'db':
+        if rgid:
+            return return_rg(rsid, rgid)
+    else:
+        if not path:
+            path = 'app/main/static/rule_config.yml'
+        return yaml.load(open(path), Loader=yaml.FullLoader), None
 
 
-def write_validations(data):
-    try:
-        data = data.loc[~data['validation']]
-        obj = []
-        data.apply(lambda x: obj.append(Validation(rid=x['rid'], rgid=x['rgid'], modified_date=x['modified_date'],
-                                           responsible=x['responsible'], created_date=x['created_date'])), axis=1)
-        db.session.bulk_save_objects(obj)
-    except Exception as e:
-        return str(e)
+def write_validations(data, where='file', path=None):
+    if where == 'db':
+        try:
+            # data = data.loc[~data['validation']]
+            obj = []
+            data.apply(lambda x: obj.append(Validation(rid=x['rid'], rgid=x['rgid'], modified_date=x['modified_date'],
+                                            responsible=x['responsible'], created_date=x['created_date'])), axis=1)
+            db.session.bulk_save_objects(obj)
+            db.session.commit()
+        except Exception as e:
+            return str(e)
+    else:
+        try:
+            if not path:
+                path = 'validation.xlsx'
+            data.to_excel(path)
+        except Exception as e:
+            return str(e)
     return 'success'

app/main/utils/db_util.py

 from app import db
-from app.models import Ruleset
+from app.models import Ruleset, RuleGroup
+
+
+def add_rs(config):
+    rs = Ruleset()
+    rs.from_dict(config)
+    db.session.add(rs)
+    db.session.commit()
+    return str(rs.uid)
 
 
 def add_rg(rsid, rg):
     rs = db.session.query(Ruleset).filter(Ruleset.uid == rsid).one()
+    l = len(rs.rulegroups)
     rs.add_rulegroup(rg)
     db.session.add(rs)
     db.session.commit()
-    return True
+    return str(rs.rulegroups[l].uid)
 
 
 def return_rg(rsid, rgid=None):
     rs = db.session.query(Ruleset).filter(Ruleset.uid == rsid).one()
-    rules = rs.return_rulegroups()
+    rgs = rs.return_rulegroups()
     if rgid:
-        rules = list(filter(lambda x: x.uid == rgid, rules))
-    return rules
+        try:
+            rule = db.session.query(RuleGroup).filter(RuleGroup.uid == rgid, RuleGroup.rsid == rsid).one()
+        except Exception:
+            raise Exception('rulegroup not found')
+        rgs = {'name': rule.name, 'rules': rule.rules}
+    return rgs, rs
 
 
 def rm_rg(rsid, rgid=None):

app/main/views/rule_group.py → app/main/views/rulegroup.py

@@ -3,18 +3,19 @@ from app.main.utils.common import response_message
 from flask import request
 from rulemall import RulesGroup
 from app.main.utils.db_util import add_rg, return_rg, rm_rg
+from datetime import datetime
 
 
 @main.post('/rg/<uuid:rsid>')
 def create_rg(rsid):
     req = request.get_json()
     try:
-        configs = {'name': req['name'], 'rules': req['rules']}
+        configs = {'name': req['name'], 'rules': req['rules'], 'created_date': str(datetime.now()),
+                   'modified_date': str(datetime.now())}
         rg = RulesGroup().from_dict(configs)
-        add_rg(rsid, configs)
+        return response_message(add_rg(rsid, configs))
     except Exception as e:
         return response_message(str(e), 500)
-    return response_message()
 
 
 @main.get('/rg/<uuid:rsid>')
@@ -36,6 +37,7 @@ def read_rg(rsid, rgid):
     return response_message(rgs)
 
 
+# TODO
 @main.put('/rg')
 def update_rg():
     req = request.get_json()
@@ -44,7 +46,7 @@ def update_rg():
 
 
 @main.delete('/rg/<uuid:rsid>')
-def delete_rg(rsid):
+def delete_rg_all(rsid):
     try:
         rgs = rm_rg(rsid)
     except Exception as e:

app/main/views/ruleset.py

+from app.main import main
+from app.main.utils.common import response_message
+from flask import request
+from app.main.utils.db_util import add_rs
+from datetime import datetime
+
+
+@main.post('/rs')
+def create_rs():
+    req = request.get_json()
+    try:
+        req['created_date'] = datetime.now()
+        req['modified_date'] = datetime.now()
+        return response_message(add_rs(req))
+    except Exception as e:
+        return response_message(str(e), 500)

app/main/views/validate.py

@@ -5,18 +5,19 @@ import pandas as pd
 from datetime import datetime
 
 
-@main.route('/validate/<ds>/<uid>', methods=['GET', 'POST'])
-def validate(ds, uid):
-    configs = read_rules()
+@main.route('/validate/<rsid>/<rgid>', methods=['GET', 'POST'])
+def validate(rsid, rgid):
+    configs, rs = read_rules(rsid, rgid, where='db')
     rg = RulesGroup().from_dict(configs)
-    cols = rg.get_target()
-    data = read_data(ds, uid, cols)
+    # TODO fix get target
+    # cols = rg.get_target()
+    data = read_data(rsid, rs)
     rg.reset_data(data)
     validation, responsible = rg.validate()
     validation = pd.concat([validation.rename('validation'), responsible], axis=1)
     validation = validation.reset_index().rename(columns={'index': 'rid'})
     # TODO return rule group id
-    validation['rgid'] = rg.id
+    validation['rgid'] = rgid
     validation['created_date'] = datetime.now()
     validation['modified_date'] = datetime.now()
     result = write_validations(validation)

app/models.py

@@ -15,7 +15,7 @@ class Ruleset(db.Model):
         return f'<Ruleset {self.uid}>'
 
     def from_dict(self, dct):
-        for field in ['uid', 'created_date', 'modified_date', 'query', 'connection_string']:
+        for field in ['data', 'created_date', 'modified_date']:
             if field in dct:
                 setattr(self, field, dct[field])
         if 'rulegroups' in dct:
@@ -63,9 +63,10 @@ class RuleGroup(db.Model):
         return f'<RuleGroup {self.uid}>'
 
     def from_dict(self, dct):
-        for field in ['name', 'rules']:
+        for field in ['name', 'rules', 'created_date', 'modified_date']:
             if field in dct:
                 setattr(self, field, dct[field])
+        return self
 
     def to_dict(self):
         dct = {'uid': self.uid, 'rsid': self.rsid, 'name': self.name, 'created_date': self.created_date,