modifications

app/main/__init__.py

@@ -6,5 +6,5 @@ main = Blueprint('main', __name__)
 CORS(main)
 
 from app.main import errors
-from app.main.views import life_check, validate, rulegroup, ruleset
+from app.main.views import life_check, validate, validators, crud
 from app import db

app/main/utils/common.py

@@ -9,7 +9,9 @@ from app import db
 from app.models import Validation
 import yaml
 import requests
-from app.main.utils.db_util import return_rg
+from app.main.utils.db_util import return_obj
+from sqlalchemy import create_engine
+from collections.abc import MutableMapping
 
 
 def response_message(data=None, status=200):
@@ -60,28 +62,23 @@ def convert(o):
     json.dumps(o)
 
 
-def read_data(puid=None, ruleset=None, cols=[], path=None):
-    if puid:
-        if ruleset:
-            ruleset.data['type'] = 'rp'
-            uid = ruleset.data['uid']
-        else:
-            uid = puid
-        if cols:
-            req = requests.post(f'{current_app.config["IO"]}/get_data/{uid}', json={'columns': cols, 'where': 'rules'})
-        else:
-            req = requests.post(f'{current_app.config["IO"]}/get_data/{uid}', json={'form_code': 'all', 'where': 'rules'})
+def read_data(dsid):
+    dataset = return_obj(dsid, 'ds', to_dict=False)
+    if dataset.data['type'] == 'rp':
+        uid = dataset.data['uid']
+        req = requests.post(f'{current_app.config["IO"]}/get_data/{uid}', json={'form_code': 'all', 'where': 'rules'})
         return pd.DataFrame(req.json()['data'])
-    else:
-        if not path:
-            path = 'app/main/static/sample_data.xlsx'
-        return pd.read_excel(path)
+    if dataset.data['type'] == 'db':
+        query = dataset.data['query']
+        conn = dataset.data['connection_string']
+        engine = create_engine(conn)
+        return pd.read_sql(query, engine)
 
 
-def read_rules(rsid=None, rgid=None, where='file', path=None):
+def read_rules(rgid=None, where='file', path=None):
     if where == 'db':
         if rgid:
-            return return_rg(rsid, rgid)
+            return return_obj(rgid, 'rg')
     else:
         if not path:
             path = 'app/main/static/rule_config.yml'
@@ -90,15 +87,16 @@ def read_rules(rsid=None, rgid=None, where='file', path=None):
 
 def write_validations(data, where='file', path=None):
     if where == 'db':
-        try:
+        # try:
             # data = data.loc[~data['validation']]
-            obj = []
-            data.apply(lambda x: obj.append(Validation(rid=x['rid'], rgid=x['rgid'], modified_date=x['modified_date'],
-                                            responsible=x['responsible'], created_date=x['created_date'])), axis=1)
-            db.session.bulk_save_objects(obj)
-            db.session.commit()
-        except Exception as e:
-            return str(e)
+        obj = []
+        data.apply(lambda x: obj.append(Validation(rid=x['rid'], rgid=x['rgid'], modified_date=x['modified_date'],
+                                                   validated=x['validated'], responsible=x['responsible'],
+                                                   created_date=x['created_date'], dsid=x['dsid'])), axis=1)
+        db.session.bulk_save_objects(obj)
+        db.session.commit()
+        # except Exception as e:
+        #     return str(e)
     else:
         try:
             if not path:

app/main/utils/db_util.py

 from app import db
-from app.models import Ruleset, RuleGroup
+from app.models import Dataset, RuleGroup, Rule
 
 
-def add_rs(config):
-    rs = Ruleset()
-    rs.from_dict(config)
-    db.session.add(rs)
+def add_obj(config, what):
+    dct = {'ds': Dataset, 'rg': RuleGroup, 'rule': Rule}
+    obj = dct[what]()
+    obj.from_dict(config)
+    db.session.add(obj)
     db.session.commit()
-    return str(rs.uid)
+    return str(obj.uid)
 
 
-def add_rg(rsid, rg):
-    rs = db.session.query(Ruleset).filter(Ruleset.uid == rsid).one()
-    l = len(rs.rulegroups)
-    rs.add_rulegroup(rg)
-    db.session.add(rs)
-    db.session.commit()
-    return str(rs.rulegroups[l].uid)
-
-
-def return_rg(rsid, rgid=None):
-    rs = db.session.query(Ruleset).filter(Ruleset.uid == rsid).one()
-    rgs = rs.return_rulegroups()
-    if rgid:
-        try:
-            rule = db.session.query(RuleGroup).filter(RuleGroup.uid == rgid, RuleGroup.rsid == rsid).one()
-        except Exception:
-            raise Exception('rulegroup not found')
-        rgs = {'name': rule.name, 'rules': rule.rules}
-    return rgs, rs
-
-
-def rm_rg(rsid, rgid=None):
-    rs = db.session.query(Ruleset).filter(Ruleset.uid == rsid).one()
-    if rgid:
-        rs = Ruleset()
-        rs.rulegroups = list(filter(lambda x: x.uid != rgid, rs.rulegroups))
+def return_obj(oid, what, to_dict=True):
+    dct = {'ds': Dataset, 'rg': RuleGroup, 'rule': Rule}
+    obj = db.session.query(dct[what]).filter(dct[what].uid == oid).one()
+    if to_dict:
+        return obj.to_dict()
     else:
-        db.session.delete(rs)
+        return obj
+
+
+def rm_obj(oid, what):
+    dct = {'ds': Dataset, 'rg': RuleGroup, 'rule': Rule}
+    obj = db.session.query(dct[what]).filter(dct[what].uid == oid).one()
+    db.session.delete(obj)
     db.session.commit()
-    return rsid
+    return True
+
+
+def return_project_rules(project_id):
+    rules = db.session.query(Rule).filter(Rule.project_id == project_id).all()
+    return [r.to_dict() for r in rules]

app/main/views/validate.py

@@ -5,22 +5,23 @@ import pandas as pd
 from datetime import datetime
 
 
-@main.route('/validate/<rsid>/<rgid>', methods=['GET', 'POST'])
-def validate(rsid, rgid):
-    configs, rs = read_rules(rsid, rgid, where='db')
+@main.route('/validate/<dsid>/<rgid>', methods=['GET', 'POST'])
+def validate(dsid, rgid):
+    configs = read_rules(rgid, where='db')
     print('got rules')
     rg = RulesGroup()
     rg.from_dict(configs)
     # cols = rg.get_target()
-    data = read_data(rsid, rs)
+    data = read_data(dsid)
     print('got data')
     rg.reset_data(data)
     validation, responsible = rg.validate()
     print('validated')
-    validation = pd.concat([validation.rename('validation'), responsible], axis=1)
+    validation = pd.concat([validation.rename('validated'), responsible], axis=1)
     validation = validation.reset_index().rename(columns={'index': 'rid'})
     # TODO return rule group id
     validation['rgid'] = rgid
+    validation['dsid'] = dsid
     validation['created_date'] = datetime.now()
     validation['modified_date'] = datetime.now()
     result = write_validations(validation, where='db')

app/models.py

 from app import db
 from sqlalchemy.dialects.postgresql import JSONB, UUID
-from sqlalchemy import ForeignKeyConstraint
+from sqlalchemy import ForeignKey
 from uuid import uuid4
+from datetime import datetime
+import pandas as pd
 
 
-class Ruleset(db.Model):
-    __tablename__ = 'ruleset'
+class Dataset(db.Model):
+    __tablename__ = 'dataset'
     uid = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid4)
     data = db.Column(JSONB)
-    created_date = db.Column(db.DateTime)
-    modified_date = db.Column(db.DateTime)
+    created_date = db.Column(db.DateTime, default=datetime.now())
+    modified_date = db.Column(db.DateTime, default=datetime.now())
 
     def __repr__(self):
-        return f'<Ruleset {self.uid}>'
+        return f'<Dataset {self.uid}>'
 
     def from_dict(self, dct):
         for field in ['data', 'created_date', 'modified_date']:
             if field in dct:
                 setattr(self, field, dct[field])
-        if 'rulegroups' in dct:
-            self.rulegroups = [RuleGroup().from_dict(rg) for rg in dct]
 
-    def to_dict(self, keys=['uid', 'created_date', 'modified_date', 'connection_string', 'query', 'rulegroups']):
-        dct = {'uid': self.uid, 'created_date': self.created_date, 'connection_string': self.connection_string,
-               'modified_date': self.modified_date, 'query': self.query}
+    def to_dict(self, keys=['uid', 'created_date', 'modified_date', 'data']):
+        dct = {'uid': str(self.uid), 'created_date': str(self.created_date), 'data': self.data,
+               'modified_date': str(self.modified_date)}
         out = {}
         for key in keys:
-            if key == 'rulegroups':
-                if self.rulegroups:
-                    out[key] = self.return_rulegroups()
-                else:
-                    out[key] = None
-            else:
-                out[key] = dct[key]
+            out[key] = dct[key]
         return out
 
-    def remove_rulegroups(self):
-        self.rulegroups = []
-
-    def add_rulegroup(self, rg):
-        self.rulegroups = self.rulegroups + [RuleGroup().from_dict(rg)]
-
-    def return_rulegroups(self):
-        return [rg.to_dict() for rg in self.rulegroups]
-
-    def return_rulegroup_names(self):
-        return [rg.get_name() for rg in self.rulegroups]
-
 
 class RuleGroup(db.Model):
     __tablename__ = 'rulegroup'
     uid = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid4)
-    rsid = db.Column(UUID(as_uuid=True), default=uuid4)
     name = db.Column(db.String)
     rules = db.Column(JSONB)
-    created_date = db.Column(db.DateTime)
-    modified_date = db.Column(db.DateTime)
-
-    __table_args__ = (ForeignKeyConstraint([rsid], [Ruleset.uid]),)
+    created_date = db.Column(db.DateTime, default=datetime.now())
+    modified_date = db.Column(db.DateTime, default=datetime.now())
 
     def __repr__(self):
         return f'<RuleGroup {self.uid}>'
@@ -69,28 +48,57 @@ class RuleGroup(db.Model):
         return self
 
     def to_dict(self):
-        dct = {'uid': self.uid, 'rsid': self.rsid, 'name': self.name, 'created_date': self.created_date,
-               'modified_date': self.modified_date, 'rules': self.rules}
+        dct = {'uid': str(self.uid), 'name': self.name, 'created_date': str(self.created_date), 'rules': self.rules,
+               'modified_date': str(self.modified_date)}
         return dct
 
     def get_name(self):
         return self.name
 
 
-Ruleset.rulegroups = db.relationship('RuleGroup', foreign_keys=[RuleGroup.rsid], cascade='all, delete, delete-orphan',
-                                     order_by=[RuleGroup.rsid, RuleGroup.created_date], backref='table')
-
-
 class Validation(db.Model):
     __tablename__ = 'validation'
     uid = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid4)
     rid = db.Column(db.String)
-    rgid = db.Column(UUID(as_uuid=True), default=uuid4)
+    rgid = db.Column(UUID(as_uuid=True), ForeignKey('rulegroup.uid'), default=uuid4)
+    dsid = db.Column(UUID(as_uuid=True), ForeignKey('dataset.uid'), default=uuid4)
     responsible = db.Column(JSONB)
-    created_date = db.Column(db.DateTime)
-    modified_date = db.Column(db.DateTime)
-
-    __table_args__ = (ForeignKeyConstraint([rgid], [RuleGroup.uid]),)
+    validated = db.Column(db.Boolean)
+    created_date = db.Column(db.DateTime, default=datetime.now())
+    modified_date = db.Column(db.DateTime, default=datetime.now())
 
     def __repr__(self):
         return f'<Validation {self.uid}>'
+
+
+class Rule(db.Model):
+    __tablename__ = 'rule'
+    uid = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid4)
+    rule = db.Column(JSONB)
+    name = db.Column(db.String)
+    project_id = db.Column(db.String)
+    created_date = db.Column(db.DateTime, default=datetime.now())
+    modified_date = db.Column(db.DateTime, default=datetime.now())
+
+    def from_dict(self, dct):
+        for field in ['uid', 'rule', 'project_id', 'created_date', 'modified_date']:
+            if field in dct:
+                setattr(self, field, dct[field])
+        setattr(self, 'name', dct['rule']['rule_attributes']['name'])
+        return self
+
+    def to_dict(self, flat=True):
+        dct = {'uid': str(self.uid), 'rule': self.rule, 'created_date': str(self.created_date),
+               'project_id': self.project_id, 'modified_date': str(self.modified_date), 'name': self.name}
+        if flat:
+            return self.rule_flatter(dct)
+        return dct
+
+    @staticmethod
+    def rule_flatter(unflat_dct):
+        dct = unflat_dct.copy()
+        r = dct.pop('rule')
+        rr = r.pop('rule_attributes')
+        r.update(rr)
+        dct.update(pd.json_normalize(r).to_dict(orient='records')[0])
+        return dct

make_db.py

@@ -4,7 +4,7 @@ from app import configs
 from app.models import *
 
 app = Flask(__name__)
-config_name = os.getenv('FLASK_ENV') or 'default'
+config_name = os.getenv('FLASK_ENV') or 'local'
 app.config.from_object(configs[config_name])
 db.init_app(app)
 app.app_context().push()