WIP: added CRUD functions

app/config.py

@@ -4,13 +4,14 @@ basedir = os.path.abspath(os.path.dirname(__file__))
 
 
 class Config:
+    API_DESCRIPTION = 'Broccoli Rule Engine'
+    IO = 'http://sumit_io:5300'
     DEBUG = False
     TESTING = False
-    API_DESCRIPTION = 'Broccoli Rule Engine'
     SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postgres:postgres@10.1.1.5:5432/dqdb_test'
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     LOG_TO_STDOUT = os.environ.get('LOG_TO_STDOUT')
-    PORT = 5800
+    PORT = 5200
     AUTO_UPDATE = False
     CACHE_TYPE = "SimpleCache"
     CACHE_DEFAULT_TIMEOUT = 300
@@ -38,6 +39,7 @@ class TestingConfig(Config):
 
 
 class LocalConfig(Config):
+    IO = 'http://localhost:5300'
     DEBUG = True
     AUTO_UPDATE = False
 

app/main/static/rule_config.yml

 ---
+name:
+  test1
 rules:
-  - rule_type: Unique
-    rule_attributes:
-      name: unique_customer_id
-      target: customer_id
-      orient: column
-      invert: false
-    validator_parameters:
-      keep: false
-  - rule_type: Unique
-    rule_attributes:
-      name: not_unique_loan_id
-      target: loan_id
-      orient: column
-      invert: true
-    validator_parameters:
-      keep: false
-  - rule_type: Range
-    rule_attributes:
-      name: age_range_18_50
-      target: age
-      orient: column
-    validator_parameters:
-      maximum: 50
-      minimum: 18
-      left_inclusive: true
-      right_inclusive: true
-  - rule_type: Category
-    rule_attributes:
-      name: cutomer_type_category
-      target: customer_type
-      orient: column
-    validator_parameters:
-      categories:
-        - 1
-        - 2
-        - 3
-        - 4
-        - 5
-  - rule_type: Range
-    rule_attributes:
-      name: age_range_under_18
-      target: age
-      orient: column
-    validator_parameters:
-      minimum: 18
-
-  - rule_type: Length
-    rule_attributes:
-      name: cellphone_length_valid
-      target: cell_phone
-      orient: column
-      invert: true
-    validator_parameters:
-      min_len: 11
-      max_len: 11
-      left_inclusive: true
-      right_inclusive: true
-
-  - rule_type: Category
-    rule_attributes:
-      name: military_service
-      target: military_service_status
-      orient: column
-    validator_parameters:
-      categories:
-        - true
+  AND:
+    - rule_type: Unique
+      rule_attributes:
+        name: unique_customer_id
+        target: customer_id
+        orient: column
+        invert: false
+      validator_parameters:
+        keep: false
+    - rule_type: Unique
+      rule_attributes:
+        name: not_unique_loan_id
+        target: loan_id
+        orient: column
+        invert: true
+      validator_parameters:
+        keep: false
+    - rule_type: Range
+      rule_attributes:
+        name: age_range_18_50
+        target: age
+        orient: column
+      validator_parameters:
+        maximum: 50
+        minimum: 18
+        left_inclusive: true
+        right_inclusive: true
+    - rule_type: Category
+      rule_attributes:
+        name: cutomer_type_category
+        target: customer_type
+        orient: column
+      validator_parameters:
+        categories:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+    - rule_type: Range
+      rule_attributes:
+        name: age_range_under_18
+        target: age
+        orient: column
+      validator_parameters:
+        minimum: 18
+    - rule_type: Length
+      rule_attributes:
+        name: cellphone_length_valid
+        target: cell_phone
+        orient: column
+        invert: true
+      validator_parameters:
+        min_len: 11
+        max_len: 11
+        left_inclusive: true
+        right_inclusive: true
+    - rule_type: Category
+      rule_attributes:
+        name: military_service
+        target: military_service_status
+        orient: column
+      validator_parameters:
+        categories:
+          - true
 

app/main/utils/common.py

 import gzip
 import collections
-from flask import make_response, json
+from flask import make_response, json, current_app
 import numpy as np
 import pandas as pd
 from app.errors import *
@@ -9,6 +9,7 @@ from rulemall.utils.rule_builder import rule_builder
 from app import db
 from app.models import Validation
 import yaml
+import requests
 
 
 def response_message(data=None, status=200):
@@ -59,13 +60,17 @@ def convert(o):
     json.dumps(o)
 
 
-def read_data():
-    return pd.read_excel('app/main/static/sample_data.xlsx')
+def read_data(ds=None, uid=None, cols=[]):
+    if ds:
+        req = requests.post(f'{current_app.config["IO"]}/r/data/{ds}/{uid}', json={'columns': cols})
+        return pd.DataFrame(req.json()['data'])
+    else:
+        return pd.read_excel('app/main/static/sample_data.xlsx')
 
 
 def read_rules():
     path = 'app/main/static/rule_config.yml'
-    return rule_builder(yaml.load(open(path), Loader=yaml.FullLoader))
+    return yaml.load(open(path), Loader=yaml.FullLoader)
 
 
 def write_validations(data):

app/main/views/validate.py

@@ -5,17 +5,18 @@ import pandas as pd
 from datetime import datetime
 
 
-@main.route('/validate', methods=['GET', 'POST'])
-def validate():
-    data = read_data()
+@main.route('/validate/<ds>/<uid>', methods=['GET', 'POST'])
+def validate(ds, uid):
     configs = read_rules()
-    rs = RulesGroup().from_dict(configs)
-    rs.reset_data(data)
-    validation, responsible = rs.validate()
+    rg = RulesGroup().from_dict(configs)
+    cols = rg.get_target()
+    data = read_data(ds, uid, cols)
+    rg.reset_data(data)
+    validation, responsible = rg.validate()
     validation = pd.concat([validation.rename('validation'), responsible], axis=1)
     validation = validation.reset_index().rename(columns={'index': 'rid'})
     # TODO return rule group id
-    validation['rgid'] = rs.id
+    validation['rgid'] = rg.id
     validation['created_date'] = datetime.now()
     validation['modified_date'] = datetime.now()
     result = write_validations(validation)

app/models.py

@@ -7,8 +7,7 @@ from uuid import uuid4
 class Ruleset(db.Model):
     __tablename__ = 'ruleset'
     uid = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid4)
-    query = db.Column(db.String)
-    connection_string = db.Column(db.String)
+    data = db.Column(JSONB)
     created_date = db.Column(db.DateTime)
     modified_date = db.Column(db.DateTime)