Initial Commit

Dockerfile

+# pull official base image
+FROM python:3.8-slim
+
+WORKDIR /home/pv
+
+COPY . .
+RUN pip install -r requirements.txt --no-index --find-links wheels/
+
+RUN chmod a+x boot.sh
+
+ENV FLASK_APP manage.py
+ENV FLASK_ENV production
+
+ENTRYPOINT ["./boot.sh"]

Jenkinsfile

+node {
+	    // holds reference to docker image
+        def dockerImage
+
+        // ip address of the docker private repository
+        def dockerImageTag = "pv-report"
+        def dockerNetwork = "docker_vista"
+        // Application port
+        def dockerPort = "5800"
+        def endpointPort = "5800"
+
+	    stage('Clone Repo') {
+                checkout scm
+        }
+
+
+        stage('Build Docker Image') {
+
+            echo "Building Docker Image"
+            
+            sh "docker build -t ${dockerImageTag} ."
+            //dockerImage = docker.build("${dockerImageTag}")
+        }
+
+
+        stage('Deploy Docker Image'){
+
+            echo "Docker Image Tag Name: ${dockerImageTag}"
+
+            sh "docker stop ${dockerImageTag} || true && docker rm ${dockerImageTag} || true"
+
+            sh "docker network create ${dockerNetwork} || true"
+
+            sh "docker run --name ${dockerImageTag} -d --net ${dockerNetwork} --restart always -p ${dockerPort}:${endpointPort} ${dockerImageTag}"
+        }
+
+        stage('Notify GitLab') {
+            updateGitlabCommitStatus name: 'build', state: 'pending'
+            updateGitlabCommitStatus name: 'build', state: 'success'
+       }
+}

README.md

+# SUMIT API (DBIO)
+local config:
+
+SET FLASK_ENV = local (windows)
+
+export FLASK_ENV = local (linux)
+
+python manage.py runserver

app/__init__.py

+import os
+import gc
+import logging
+from logging.handlers import RotatingFileHandler
+from flask import Flask
+from flask_caching import Cache
+from apscheduler.schedulers.background import BackgroundScheduler
+from app.config import configs
+
+cache = Cache()
+
+
+# def pipeline_runner():
+#     new_app = Flask(__name__)
+#     config_name = os.getenv('FLASK_ENV') or 'default'
+#     new_app.config.from_object(configs[config_name])
+#     if new_app.config.get('REPORT_CONFIG_FILE'):
+#         with open(new_app.config.get('REPORT_CONFIG_FILE'), 'r', encoding='utf8') as f:
+#             new_app.config['REPORT_CONFIG'] = yaml.load(f, SafeLoader)
+#             f.close()
+#     with new_app.app_context():
+#         return transfer_pipeline()
+
+
+def create_app():
+    app = Flask(__name__)
+
+    config_name = os.getenv('FLASK_ENV') or 'default'
+    app.config.from_object(configs[config_name])
+
+    cache.init_app(app)
+
+    from app.main import main as main_blueprint
+    app.register_blueprint(main_blueprint)
+
+    # if app.config['AUTO_UPDATE']:
+    #     sched = BackgroundScheduler(daemon=True)
+    #     sched.add_job(pipeline_runner, 'interval', seconds=90)
+    #     sched.start()
+
+    if not app.debug and not app.testing:
+        if app.config['LOG_TO_STDOUT']:
+            stream_handler = logging.StreamHandler()
+            stream_handler.setLevel(logging.INFO)
+            app.logger.addHandler(stream_handler)
+        else:
+            if not os.path.exists('logs'):
+                os.mkdir('logs')
+            file_handler = RotatingFileHandler('logs/reporting.log', maxBytes=10240, backupCount=10)
+            file_handler.setFormatter(logging.Formatter(
+                '%(asctime)s %(levelname)s: %(message)s '
+                '[in %(pathname)s:%(lineno)d]'))
+            file_handler.setLevel(logging.INFO)
+            app.logger.addHandler(file_handler)
+
+        app.logger.setLevel(logging.INFO)
+        app.logger.info('PV Reporting Server Created')
+
+    gc.collect()
+    return app

app/config.py

+import os
+
+basedir = os.path.abspath(os.path.dirname(__file__))
+
+
+class Config:
+    DEBUG = False
+    TESTING = False
+    API_DESCRIPTION = 'Broccoli Rule Engine'
+    SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postges:postges@10.1.1.5:5432/dqdb_test'
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    LOG_TO_STDOUT = os.environ.get('LOG_TO_STDOUT')
+    PORT = 5800
+    AUTO_UPDATE = False
+    CACHE_TYPE = "SimpleCache"
+    CACHE_DEFAULT_TIMEOUT = 300
+
+
+class DevelopmentConfig(Config):
+    """
+    Development configurations
+    """
+    DEBUG = False
+    SQLALCHEMY_ECHO = True
+    ASSETS_DEBUG = False
+
+
+class ProductionConfig(Config):
+    """
+    Production configurations
+    """
+    DEBUG = False
+    AUTO_UPDATE = True
+
+
+class TestingConfig(Config):
+    TESTING = True
+
+
+class LocalConfig(Config):
+    DEBUG = True
+    AUTO_UPDATE = False
+
+
+configs = {
+    'development': DevelopmentConfig(),
+    'testing': TestingConfig(),
+    'production': ProductionConfig(),
+    'default': LocalConfig(),
+    'local': LocalConfig()
+}

app/errors/__init__.py

+"""
+SUMIT Public Exceptions and Warnings
+"""
+
+
+class JsonError(Exception):
+    """Exception raised for errors envolving json."""
+
+    def __init__(self, expression, type=False):
+        if type:
+            self.message_en = f'{expression} is not a valid json'
+            self.message_fa = f'{expression} جیسون معتبری نیست'
+        else:
+            self.message_en = f"{expression} can't be jsonified"
+            self.message_fa = f'امکان تبدیل {expression} به جیسون نیست'
+
+
+class EmptyError(Exception):
+    """Exception raised for errors envolving empty data/column."""
+
+    def __init__(self, expression='', table=True, common=False, new=False):
+        if table:
+            self.message_en = f'data is empty'
+            self.message_fa = f'داده خالی است'
+        elif not common:
+            self.message_en = f'column {expression} is empty'
+            self.message_fa = f'ستون {expression} خالی است'
+        elif new:
+            self.message_en = 'No new data found'
+            self.message_fa = 'داده‌ی جدیدی وجود ندارد'
+        else:
+            self.message_en = f'there is no common data for specified columns'
+            self.message_fa = f'هیچ داده‌ی مشترکی برای ستون‌های انتخابی موجود نیست'
+
+
+class ArgsError(Exception):
+    """Exception raised for errors with args."""
+
+    def __init__(self, expression, valid=True):
+        if valid:
+            self.message_en = f'{expression} should be specified'
+            self.message_fa = f'{expression} باید مشخص شود'
+        else:
+            self.message_en = f'{expression} specified is not valid'
+            self.message_fa = f'{expression}مشخص‌شده معتبر نیست '
+
+
+class DtypeError(Exception):
+    """Exception raised for errors in dtype."""
+
+    def __init__(self, expression, all=False, col=False):
+        if all:
+            self.message_en = f'no {expression} column in data'
+            self.message_fa = f' هیچ ستونی با تایپ{expression} در داده وجود ندارد'
+        elif not col:
+            self.message_en = f'all columns should have {expression} dtype'
+            self.message_fa = f'تمامی ستون‌های انتخابی باید تایپ{expression} داشته باشند'
+        else:
+            self.message_en = f"columns {expression} doesn't have the same dtype as specified"
+            self.message_fa = f'ستون {expression} دارای تایپ مشخص‌شده نیست '
+
+
+class ClusterError(Exception):
+    """Exception raised for errors in cluster/neighbor numbers."""
+
+    def __init__(self):
+        self.message_en = f'Number of rows with data is less than number of neighbors/clusters'
+        self.message_fa = f'تعداد داده‌ها از تعداد همسایه‌ها/کلاسترهای انتخابی کمتر است'
+
+
+class MethodError(Exception):
+    """Exception raised for errors in the method."""
+
+    def __init__(self, where=None, method=None):
+        if not where:
+            self.message_en = 'There was an error, please try again'
+            self.message_fa = 'خطایی پیش آمده، لطفا مجددا تلاش کنید'
+        if where == 'anova':
+            if method == 'one_way':
+                self.message_en = 'In one_way method length of columns is 1'
+                self.message_fa = 'در روش one_way تعداد ستون باید ۱ باشد'
+            elif method == 'two_way':
+                self.message_en = 'In two_way method lenght of columns is 2, length of columns2 is 1 and there is no common columns between them'
+                self.message_fa = 'در روش two_way تعداد ستون‌ها در شاخص اول ۲، شاخص دوم ۱ است و هیچ اشتراکی بین شاخص‌ها نباید باشد'
+            else:
+                self.message_en = 'In manova lenghth of columns is more than 1 and lenght of columns2 is more than 0'
+                self.message_fa = 'در روش manova تعداد ستون‌ها در شاخص اول باید بیش از ۱ و در شاخص دوم بیش ۰ باشد'
+        elif where == 'contingency':
+            self.message_en = 'Please Specify two columns'
+            self.message_fa = 'لطفا دو شاخص انتخاب کنید'
+        elif where == 'qq':
+            self.message_en = 'In on_way method 1 columns and in two_way 2 columns should be specified'
+            self.message_fa = 'در روش on_way یک ستون و در روش two_way دو ستون باید مشخص شود'
+        elif where == 'ttest':
+            self.message_en = 'In singular method 1 columns and in dependent and independent methods 2 columns should be specified'
+            self.message_fa = 'در روش singular یک ستون و در روش‌های dependent و independent دو ستون باید مشخص شود'
+        elif where == 'regression':
+            self.message_en = 'In Regression lenghth of columns & columns2 should be more than 0'
+            self.message_fa = 'در روش رگرسیون تعداد ستون‌ها در شاخص اول و دوم باید بیش از ۰ باشد'
+
+
+class DBError(Exception):
+    """Exception raised for errors in database."""
+
+    def __init__(self):
+        self.message_en = 'Error with db'
+        self.message_fa = 'مشکل در دیتابیس'
+
+
+class ExistError(Exception):
+    """Exception raised for errors envolving s.th's existance."""
+
+    def __init__(self, expression='', col=False, intersection=False):
+        if not col:
+            self.message_en = f"{expression} doesn't exist"
+            self.message_fa = f'{expression} وجود ندارد'
+        else:
+            self.message_en = f"column {expression} doesn't exist"
+            self.message_fa = f'ستون {expression} وجود ندارد'

app/main/__init__.py

+from flask import Blueprint
+from flask_cors import CORS
+
+main = Blueprint('main', __name__)
+
+CORS(main)
+
+from app.main import errors
+from app.main.views import life_check

app/main/errors.py

+from . import main
+import gc
+
+
+@main.app_errorhandler(404)
+def page_not_found(e):
+	gc.collect()
+	return 'Page not found', 404
+
+
+@main.app_errorhandler(500)
+def internal_server_error(e):
+	gc.collect()
+	return 'Internal Server Error', 500

app/main/utils/common.py

+import gzip
+import collections
+from flask import make_response, json, current_app
+import numpy as np
+import pandas as pd
+import khayyam as kym
+from app.errors import *
+import gc
+import requests
+
+
+def response_message(data=None, status=200):
+    if status != 200:
+        try:
+            error = {'fa': data.message_fa, 'en': data.message_en}
+        except AttributeError:
+            print(data)
+            error = MethodError()
+            error = {'message_fa': error.message_fa, 'message_en': error.message_en}
+        resp_dict = {'message': error}
+    else:
+        resp_dict = data
+    resp_dict = json.loads(json.dumps(resp_dict, default=convert, sort_keys=False).replace('"None"', 'null')
+                           .replace('NaN', 'null'), strict=False)
+    content = gzip.compress(json.dumps(resp_dict, ensure_ascii=False, indent=3, default=convert,
+                                       sort_keys=False).encode('utf8'), 5)
+    response = make_response(content, status)
+    response.headers['Content-length'] = len(content)
+    response.headers['Content-Encoding'] = 'gzip'
+    response.headers['Content-Type'] = 'application/json; charset=utf-8'
+    response.headers.add('Access-Control-Allow-Origin', '*')
+    gc.collect()
+    return response
+
+
+def deep_dict_update(d, u):
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = deep_dict_update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
+def convert(o):
+    if isinstance(o, np.int64):
+        return int(o)
+    if isinstance(o, np.bool_):
+        if o:
+            return True
+        else:
+            return False
+    if pd.isna(o):
+        return None
+    if o is np.ma.masked:
+        return None
+    json.dumps(o)

app/tests/basic_tests.py

+import unittest
+from flask import current_app
+from app import create_app
+
+
+class BasicsTestCase(unittest.TestCase):
+	def setUp(self):
+		self.app = create_app('testing')
+		self.app_context = self.app.app_context()
+		self.app_context.push()
+		# db.create_all()
+
+	def tearDown(self):
+		self.app_context.pop()
+		# 	db.session.remove()
+		# 	db.drop_all()
+
+	def test_app_exists(self):
+		self.assertFalse(current_app is None)
+
+	def test_app_is_testing(self):
+		self.assertTrue(current_app.config['TESTING'])

boot.sh

+#!/bin/bash
+
+exec gunicorn -b :5800 --access-logfile - --error-logfile - manage:app

manage.py

+from app import create_app
+
+app = create_app()
+
+
+@app.shell_context_processor
+def make_shell_context():
+    return {}
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=app.config['PORT'], threaded=True)

requirements.txt

+APScheduler==3.9.1
+Flask==2.1.1
+Flask-Caching==2.0.0
+Flask-Cors==3.0.10
+Flask-SQLAlchemy==2.5.1
+gunicorn==20.1.0
+pandas==1.4.2
+psycopg2-binary==2.9.3
+requests==2.28.1
+six==1.16.0
+SQLAlchemy==1.4.35
+tzdata==2022.1
+tzlocal==4.2
+urllib3==1.26.10
+zipp==3.8.0