Skip to content

K
kl_task8_aggregations
Commit cf6f53a1 by arun.uday
Options

commit6

parent 2e4d47af
Showing with 184 additions and 94 deletions
app.py
# using mongo aggregation operations
import uvicorn
from scripts.config.applications_config import uvicorn_app, uvicorn_port
from scripts.config.applications_config import uvicorn_port
from scripts.logging.loggers import logger
# starting the application
if __name__ == "__main__":
print("Mongo Aggregations task")
uvicorn.run(uvicorn_app, port=int(uvicorn_port))
try:
print("Mongo Aggregations task")
uvicorn.run("main:app", port=int(uvicorn_port))
except Exception as e:
logger.error("Interruption occurred: ", e)
# pip install python-multipart
conf/applications.conf
[path]
base_path = scripts/
sub_path = temp/
log_path = log/
[file]
file_name_xlsx = data.xlsx
......@@ -9,7 +10,10 @@ file_name_xlsx = data.xlsx
mongodb = mongodb://localhost:27017
[mongodb]
project = $project
projection = $project
db_name = violations_logs
collection_name = logs
[uvicorn]
uvicorn_port = 8000
......
main.py 0 → 100644
# using mongo aggregation operations
import uvicorn
from fastapi import FastAPI
from scripts.config.applications_config import uvicorn_port
from scripts.logging.loggers import logger
from scripts.services.app_services_run import business_violate
app = FastAPI()
app.include_router(business_violate)
# starting the application
if __name__ == "__main__":
try:
print("Mongo Aggregations task")
uvicorn.run(app, port=int(uvicorn_port))
except Exception as e:
logger.error("Interruption occurred: ", e)
# pip install python-multipart
scripts/config/applications_config.py
......@@ -9,6 +9,7 @@ config.read("conf/applications.conf")
# path
base_path = config.get("path", 'base_path')
sub_path = config.get("path", "sub_path")
log_path = config.get("path", "log_path")
# uvicorn
uvicorn_port = config.get("uvicorn", "uvicorn_port")
......@@ -22,4 +23,6 @@ full_path_xlsx = base_path + sub_path + file_name_xlsx
client_connect = config.get("connection", "mongodb")
# mongo
project = config.get("mongodb", "project")
projection = config.get("mongodb", "projection")
db_name = config.get("mongodb", "db_name")
collection_name = config.get("mongodb", "collection_name")
scripts/core/handlers/api_read.py 0 → 100644
from scripts.config.applications_config import full_path_xlsx
from scripts.core.handlers.generate_excel_file import generate_excel
from scripts.core.handlers.mongo_db_operations import MongoDbStart
from scripts.core.handlers.uploaded_file_ import extract_uploaded_data
from scripts.logging.loggers import logger
def read_data_from_api(file_data):
try:
decoded_data = extract_uploaded_data(file_data)
db_class = MongoDbStart()
db_created = db_class.mongo_create()
db_class.mongo_insert(db_created, decoded_data)
print("\nBusiness Name with most Violations")
db_class.mongo_violation(db_created)
print("\nBusiness Name with No Violations")
db_class.mongo_no_violation(db_created)
print("\nGenerating Excel file")
cursor_data = db_class.mongo_view_results(db_created)
generate_excel(full_path_xlsx, cursor_data)
except Exception as e:
logger.error("Exception occurred while submitting: ", e)
scripts/core/handlers/data_insert.py
# inserting data to mongo
from scripts.logging.loggers import logger
def insert_data_db(db, decoded):
try:
insert_res = db.insert_many(decoded)
except Exception as e:
print("Exception occurred :", e)
logger.error("Exception occurred while inserting data: ", e)
else:
return insert_res
scripts/core/handlers/generate_excel_file.py
# generating the excel file
# generating the Excel file
import pandas as pd
from scripts.logging.loggers import logger
def generate_excel(path_xlsx, cursor_data):
data_list = []
if cursor_data.alive > 0:
for business_names in cursor_data:
data_list.append(business_names)
data_frame = pd.DataFrame(data_list, columns=['result', 'business_name', 'date'])
print(data_frame)
data_frame.to_excel(path_xlsx, index=False)
else:
print("Error")
try:
data_list = []
if cursor_data.alive > 0:
for business_names in cursor_data:
data_list.append(business_names)
data_frame = pd.DataFrame(data_list, columns=['result', 'business_name', 'date'])
print(data_frame)
data_frame.to_excel(path_xlsx, index=False)
else:
print("Error")
except Exception as e:
logger.error("Exception occurred while generating excel: ", e)
scripts/services/mongo_db_operations.py → scripts/core/handlers/mongo_db_operations.py
# class for mongodb operations
from scripts.core.engine.mongodb_connect import client
from scripts.config.applications_config import db_name, collection_name
from scripts.core.database.mongodb_connect import client
from scripts.core.handlers.data_insert import insert_data_db
from scripts.core.handlers.no_violations_data import db_select_no_violate
from scripts.core.handlers.result_generate import results_data
from scripts.core.handlers.violation_find import db_select_violate
from scripts.logging.loggers import logger
class MongoDbStart:
......@@ -13,11 +15,11 @@ class MongoDbStart:
try:
client_con = client
except Exception as e:
print("Connection failed: ", e)
logger.error("Exception occurred while connecting to mongo: ", e)
else:
db = client_con['violationslogs']
log_name = db['log']
return log_name
db = client_con[db_name]
log_name = db[collection_name]
return log_name
# mongo data insert
@staticmethod
......@@ -30,7 +32,7 @@ class MongoDbStart:
else:
print("Something went wrong data cannot be inserted....")
except Exception as e:
print("Insertion failed", e)
logger.error("Exception occurred while inserting to collection: ", e)
# mongo aggregate find businessman with most violations
@staticmethod
......@@ -38,7 +40,7 @@ class MongoDbStart:
try:
db_select_violate(db)
except Exception as e:
print("fetching failed", e)
logger.error("Exception occurred while fetching violations: ", e)
# mongo aggregate find businessman with no violations
@staticmethod
......@@ -46,7 +48,7 @@ class MongoDbStart:
try:
db_select_no_violate(db)
except Exception as e:
print("fetching failed", e)
logger.error("Exception occurred while fetching for no violations: ", e)
# mongo aggregate find
@staticmethod
......@@ -55,7 +57,4 @@ class MongoDbStart:
data = results_data(db)
return data
except Exception as e:
print("fetching failed", e)
logger.error("Exception occurred while fetching all: ", e)
scripts/core/handlers/no_violations_data.py
# business man with no violations
from scripts.logging.loggers import logger
from scripts.utils.mongo_operations import MongoDb
def db_select_no_violate(db):
mongo_obj = MongoDb()
cursor_data = mongo_obj.no_violations_db(db)
if cursor_data.alive > 0:
print("Data collected....")
for business_names in cursor_data:
print(business_names)
else:
print("Data fetching failed")
try:
mongo_obj = MongoDb()
cursor_data = mongo_obj.no_violations_db(db)
if cursor_data.alive > 0:
print("Data collected....")
for business_names in cursor_data:
print(business_names)
else:
print("Data fetching failed")
except Exception as e:
logger.error("Exception occurred while checking for violations: ", e)
scripts/core/handlers/result_generate.py
# viewing the business man and date based on results
# viewing the businessman and date based on results
from scripts.logging.loggers import logger
from scripts.utils.mongo_operations import MongoDb
def results_data(db):
mongo_obj = MongoDb()
cursor_data = mongo_obj.view_db_data(db)
if cursor_data.alive > 0:
print("Data collected....")
return cursor_data
else:
print("Data fetching failed")
return None
try:
mongo_obj = MongoDb()
cursor_data = mongo_obj.view_db_data(db)
if cursor_data.alive > 0:
print("Data collected....")
return cursor_data
else:
print("Data fetching failed")
return None
except Exception as e:
logger.error("Exception occurred while fetching results: ", e)
scripts/core/handlers/uploaded_file_.py
# file upload to csv
import json
from scripts.logging.loggers import logger
def extract_uploaded_data(file_data):
decode = file_data.decode()
decoded_data_json = json.loads(decode)
return decoded_data_json
try:
decode = file_data.decode()
decoded_data_json = json.loads(decode)
return decoded_data_json
except Exception as e:
logger.error("Exception occurred while extracting uploaded data: ", e)
scripts/core/handlers/violation_find.py
# business man with violations
# businessman with violations
from scripts.logging.loggers import logger
from scripts.utils.mongo_operations import MongoDb
def db_select_violate(db):
mongo_obj = MongoDb()
cursor_data = mongo_obj.violations_db(db)
if cursor_data.alive > 0:
print("Data collected....")
print(cursor_data.next())
else:
print("Data fetching failed")
try:
mongo_obj = MongoDb()
cursor_data = mongo_obj.violations_db(db)
if cursor_data.alive > 0:
print("Data collected....")
print(cursor_data.next())
else:
print("Data fetching failed")
except Exception as e:
logger.error("Exception occurred while checking for violations: ", e)
scripts/logging/loggers.py 0 → 100644
import logging
import os
from logging.handlers import RotatingFileHandler
from scripts.config import applications_config
def get_logger():
"""
Creates a rotating log
"""
__logger__ = logging.getLogger('')
__logger__.setLevel(logging.INFO)
log_formatter = '%(asctime)s - %(levelname)-6s - %(message)s'
time_format = "%Y-%m-%d %H:%M:%S"
file_path = applications_config.log_path
formatter = logging.Formatter(log_formatter, time_format)
if not os.path.exists(file_path):
os.makedirs(file_path)
log_file = os.path.join(f"{file_path}{applications_config.db_name}.log")
temp_handler = RotatingFileHandler(log_file, maxBytes=1)
temp_handler.setFormatter(formatter)
__logger__.addHandler(temp_handler)
return __logger__
logger = get_logger()
scripts/services/app_services_run.py
from fastapi import FastAPI, Request, UploadFile, File
from fastapi import APIRouter, Request, UploadFile, File
from fastapi.templating import Jinja2Templates
from fastapi.responses import HTMLResponse
from scripts.config.api_routes_config import route_index, route_submit
from scripts.config.applications_config import full_path_xlsx
from scripts.core.handlers.generate_excel_file import generate_excel
from scripts.core.handlers.uploaded_file_ import extract_uploaded_data
from scripts.services.mongo_db_operations import MongoDbStart
from scripts.core.handlers.api_read import read_data_from_api
from scripts.logging.loggers import logger
app = FastAPI()
business_violate = APIRouter()
templates = Jinja2Templates("scripts/core/handlers/")
# html form access using api
@app.get(route_index, response_class=HTMLResponse)
@business_violate.get(route_index, response_class=HTMLResponse)
def template_render(request: Request):
return templates.TemplateResponse("upload_file.html", {"request": request})
try:
return templates.TemplateResponse("upload_file.html", {"request": request})
except Exception as e:
logger.error("Exception occurred while loading html: ", e)
# handling the form submit using api and aggregations
@app.post(route_submit)
@business_violate.post(route_submit)
async def submit_file_data(file_data: UploadFile = File(...)):
try:
data_read = await file_data.read()
decoded_data = extract_uploaded_data(data_read)
db_class = MongoDbStart()
db_created = db_class.mongo_create()
db_class.mongo_insert(db_created, decoded_data)
print("\nBusiness Name with most Violations")
db_class.mongo_violation(db_created)
print("\nBusiness Name with No Violations")
db_class.mongo_no_violation(db_created)
print("\nGenerating Excel file")
cursor_data = db_class.mongo_view_results(db_created)
generate_excel(full_path_xlsx, cursor_data)
data = await file_data.read()
read_data_from_api(data)
except Exception as e:
print(e)
logger.error("Exception occurred while submitting: ", e)
else:
return {"msg": "file uploading successful..."}
......
scripts/temp/data.xlsx
No preview for this file type
scripts/utils/mongo_operations.py
# class for mongo operations
from scripts.config.applications_config import project
from scripts.config.applications_config import projection
class MongoDb:
......@@ -29,7 +29,7 @@ class MongoDb:
}, {
'$limit': 1
}, {
project: {
projection: {
'business_name': '$_id',
'_id': 0
}
......@@ -57,7 +57,7 @@ class MongoDb:
}
}
}, {
project: {
projection: {
'business_name': '$_id',
'_id': 0
}
......@@ -70,7 +70,7 @@ class MongoDb:
def view_db_data(db):
cursor_data = db.aggregate([
{
project: {
projection: {
'_id': 0,
'result': 1,
'business_name': 1,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment