WIP: added CRUD functions

app/config.py

@@ -4,13 +4,14 @@ basedir = os.path.abspath(os.path.dirname(__file__))
 
 
 class Config:
+    API_DESCRIPTION = 'Broccoli Rule Engine'
+    IO = 'http://sumit_io:5300'
     DEBUG = False
     TESTING = False
-    API_DESCRIPTION = 'Broccoli Rule Engine'
     SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postgres:postgres@10.1.1.5:5432/dqdb_test'
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     LOG_TO_STDOUT = os.environ.get('LOG_TO_STDOUT')
-    PORT = 5800
+    PORT = 5200
     AUTO_UPDATE = False
     CACHE_TYPE = "SimpleCache"
     CACHE_DEFAULT_TIMEOUT = 300
@@ -38,6 +39,7 @@ class TestingConfig(Config):
 
 
 class LocalConfig(Config):
+    IO = 'http://localhost:5300'
     DEBUG = True
     AUTO_UPDATE = False
 

app/main/static/rule_config.yml

 ---
+name:
+  test1
 rules:
+  AND:
     - rule_type: Unique
       rule_attributes:
         name: unique_customer_id
@@ -45,7 +48,6 @@ rules:
         orient: column
       validator_parameters:
         minimum: 18
-
     - rule_type: Length
       rule_attributes:
         name: cellphone_length_valid
@@ -57,7 +59,6 @@ rules:
         max_len: 11
         left_inclusive: true
         right_inclusive: true
-
     - rule_type: Category
       rule_attributes:
         name: military_service

app/main/utils/common.py

 import gzip
 import collections
-from flask import make_response, json
+from flask import make_response, json, current_app
 import numpy as np
 import pandas as pd
 from app.errors import *
@@ -9,6 +9,7 @@ from rulemall.utils.rule_builder import rule_builder
 from app import db
 from app.models import Validation
 import yaml
+import requests
 
 
 def response_message(data=None, status=200):
@@ -59,13 +60,17 @@ def convert(o):
     json.dumps(o)
 
 
-def read_data():
+def read_data(ds=None, uid=None, cols=[]):
+    if ds:
+        req = requests.post(f'{current_app.config["IO"]}/r/data/{ds}/{uid}', json={'columns': cols})
+        return pd.DataFrame(req.json()['data'])
+    else:
         return pd.read_excel('app/main/static/sample_data.xlsx')
 
 
 def read_rules():
     path = 'app/main/static/rule_config.yml'
-    return rule_builder(yaml.load(open(path), Loader=yaml.FullLoader))
+    return yaml.load(open(path), Loader=yaml.FullLoader)
 
 
 def write_validations(data):

app/main/views/validate.py

@@ -5,17 +5,18 @@ import pandas as pd
 from datetime import datetime
 
 
-@main.route('/validate', methods=['GET', 'POST'])
-def validate():
-    data = read_data()
+@main.route('/validate/<ds>/<uid>', methods=['GET', 'POST'])
+def validate(ds, uid):
     configs = read_rules()
-    rs = RulesGroup().from_dict(configs)
-    rs.reset_data(data)
-    validation, responsible = rs.validate()
+    rg = RulesGroup().from_dict(configs)
+    cols = rg.get_target()
+    data = read_data(ds, uid, cols)
+    rg.reset_data(data)
+    validation, responsible = rg.validate()
     validation = pd.concat([validation.rename('validation'), responsible], axis=1)
     validation = validation.reset_index().rename(columns={'index': 'rid'})
     # TODO return rule group id
-    validation['rgid'] = rs.id
+    validation['rgid'] = rg.id
     validation['created_date'] = datetime.now()
     validation['modified_date'] = datetime.now()
     result = write_validations(validation)

app/models.py

@@ -7,8 +7,7 @@ from uuid import uuid4
 class Ruleset(db.Model):
     __tablename__ = 'ruleset'
     uid = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid4)
-    query = db.Column(db.String)
-    connection_string = db.Column(db.String)
+    data = db.Column(JSONB)
     created_date = db.Column(db.DateTime)
     modified_date = db.Column(db.DateTime)