Skip to content

M
mongo_aggregate
Commit 1540973e by ajil.k
Options

updateded

parent e035cbd7
Showing with 196 additions and 166 deletions
app.py
# importing libraries
import uvicorn
from scripts.constants.application_config import port_no, link_to_fastapi
from scripts.constants.application_config import port_no
if __name__ == "__main__":
try:
uvicorn.run(link_to_fastapi, port=int(port_no), reload=True)
uvicorn.run("scripts.services.api_call:app", port=int(port_no), reload=True)
except Exception as e:
print("Error-", e)
conf/application.conf
[MongoDB]
mongo_uri = mongodb://localhost:27017/
port = 5000
[FastAPI]
link_to_fastapi = scripts.services.main:app
\ No newline at end of file
port = 5000
\ No newline at end of file
scripts/constants/application_config.py
# Importing ConfigParser
from configparser import ConfigParser
# Read conf file
config = ConfigParser()
config.read(f"conf/application.conf")
# MongoDB URI
uri = config.get("MongoDB", "mongo_uri")
port_no = config.get("MongoDB", "port")
link_to_fastapi = config.get("FastAPI", "link_to_fastapi")
# FastAPI Port
port_no = config.get("FastAPI", "port")
scripts/constants/db_connection.py
......@@ -16,4 +16,3 @@ def db_connect():
return db
except Exception as e:
print("Error ", e)
scripts/constants/endpoints.py 0 → 100644
# Class for accessing Endpoints
class APIEndpoints:
# FastAPI endpoints
root = "/"
upload_file = "/upload-file/"
violated_company = "/violated-company"
no_violated_company = "/no-violated-company"
generate_excel_file = "/generate-excel-file"
scripts/constants/read_file.py 0 → 100644
import pandas as pd
def read_excel_file():
excel_data = pd.read_excel("temp/mongodb_data.xlsx")
print("Data Stored on excel file:\n", excel_data)
scripts/core/handlers/aggregate_operations.py → scripts/core/handlers/api_functions.py
def violated_companies(app_api, collection):
try:
@app_api.get("/violated-company")
async def violated_company():
# Importing necessary libraries
import json
import openpyxl
from scripts.constants.db_connection import db_connect
from scripts.core.handlers.insert_into_db import insert_data_to_mongodb
class operations_on_api:
def __init__(self):
self.db = db_connect()
collection_name = "tbl_company"
self.collection = self.db[collection_name]
def insert_all_data(self, file):
try:
data_frame = file.file.read()
data_frame = data_frame.decode()
uploaded_path = "temp/uploaded.json"
with open(uploaded_path, "w") as json_file:
# Write data to the file
json_file.write(data_frame)
with open(uploaded_path, "r") as uploaded_json:
# Load the data from the file
data = json.load(uploaded_json)
inserted = insert_data_to_mongodb(self.collection, data)
if inserted:
print("Data inserted to DB")
else:
print("Data insertion failed!")
return inserted
except Exception as e:
print("Data insertion error-", e)
def find_violated_company(self):
try:
pipeline = [
{
'$group': {
'_id': '$business_name',
'count': {
'$sum': 1
}
}
}, {
'$sort': {
'count': -1
}
}, {
'$limit': 1
}, {
'$project': {
'business_name': '$_id',
'_id': 0
}
}
]
business_names = collection.aggregate(pipeline)
{
"$match": {
'result': "Violation Issued"
}
}, {
'$group': {
'_id': '$business_name',
'count': {
'$sum': 1
}
}
}, {
'$sort': {
'count': -1
}
}, {
'$limit': 1
}, {
'$project': {
'business_name': '$_id',
'_id': 0
}
}
]
business_names = self.collection.aggregate(pipeline)
result = str(list(business_names))
print("Business names of companies having violation\n", result)
return {"Results": result}
except Exception as e:
print("Error-", e)
return result
except Exception as e:
print("Error fetching violated company data error: ", e)
def no_violated_companies(app_api, collection):
try:
@app_api.get("/no-violated-company")
async def no_violated_company():
def find_no_violated_company(self):
try:
pipeline = [
{
'$match': {
'result': 'No Violation Issued'
}
}, {
'$group': {
'_id': None,
'business_name': {
'$push': '$business_name'
}
}
}
]
business_names = collection.aggregate(pipeline)
{
'$match': {
'result': 'No Violation Issued'
}
}, {
'$group': {
'_id': None,
'business_name': {
'$push': '$business_name'
}
}
}
]
business_names = self.collection.aggregate(pipeline)
result = str(list(business_names))
print("Business names of companies having no violation\n", result)
return {"Results": result}
except Exception as e:
print("Error-", e)
return result
except Exception as e:
print("Error fetching not violated company data error: ", e)
def create_excel_file(self):
try:
# Fetch data from MongoDB
pipeline = [
{
'$project': {
'result': '$result',
'business_name': '$business_name',
'date': '$date',
'_id': 0
}
}
]
records = list(self.collection.aggregate(pipeline))
# Create an Excel Workbook and Worksheet
wb = openpyxl.Workbook()
ws = wb.active
# Write column headers to Excel Worksheet
column_headers = list(records[0].keys())
for i, header in enumerate(column_headers):
ws.cell(row=1, column=i + 1, value=header)
# Write data from MongoDB to Excel Worksheet
for i, record in enumerate(records):
for j, key in enumerate(record.keys()):
ws.cell(row=i + 2, column=j + 1, value=record[key])
# Save the Excel Workbook
wb.save("temp/mongodb_data.xlsx")
except Exception as e:
print("Generating Excel file error-", e)
scripts/core/handlers/create_excel_file.py deleted 100644 → 0
import openpyxl
def create_excel(app_api, collection):
try:
@app_api.get("/generate-excel-file")
async def generate_excel_file():
# Fetch data from MongoDB
pipeline = [
{
'$project': {
'result': '$result',
'business_name': '$business_name',
'date': '$date',
'_id': 0
}
}
]
records = list(collection.aggregate(pipeline))
# Create an Excel Workbook and Worksheet
wb = openpyxl.Workbook()
ws = wb.active
# Write column headers to Excel Worksheet
column_headers = list(records[0].keys())
for i, header in enumerate(column_headers):
ws.cell(row=1, column=i + 1, value=header)
# Write data from MongoDB to Excel Worksheet
for i, record in enumerate(records):
for j, key in enumerate(record.keys()):
ws.cell(row=i + 2, column=j + 1, value=record[key])
# Save the Excel Workbook
wb.save("temp/mongodb_data.xlsx")
return {"Message": "Excel file created"}
except Exception as e:
print("Error-", e)
scripts/core/handlers/upload_and_insert_data.py deleted 100644 → 0
import json
from fastapi import UploadFile
from scripts.core.handlers.insert_into_db import insert_data_to_mongodb
def upload_and_insert_into_db(app_api, collection):
@app_api.post("/upload-file/")
async def create_upload_file(file: UploadFile):
data_frame = file.file.read()
data_frame = data_frame.decode()
uploaded_path = "temp/uploaded.json"
with open(uploaded_path, "w") as json_file:
# Write data to the file
json_file.write(data_frame)
with open(uploaded_path, "r") as uploaded_json:
# Load the data from the file
data = json.load(uploaded_json)
inserted = insert_data_to_mongodb(collection, data)
if inserted:
print("Data inserted to DB")
return {"filename": file.filename}
scripts/services/api_call.py 0 → 100644
# Importing Libraries
from fastapi import FastAPI, UploadFile
from scripts.constants.endpoints import APIEndpoints
from scripts.constants.read_file import read_excel_file
from scripts.core.handlers.api_functions import operations_on_api
# Creating FastAPI object
app = FastAPI()
# Create Endpoint instance
api_endpoint = APIEndpoints()
# Root
@app.get(api_endpoint.root)
async def root():
return "It's Working!"
# upload file and insert into db
@app.post(api_endpoint.upload_file, tags=["Upload Json File"])
async def upload_file(file: UploadFile):
operations_on_api().insert_all_data(file)
return {"filename": file.filename}
# Printing Violated Companies names
@app.get(api_endpoint.violated_company, tags=["Get Violated Companies"])
async def violated_company():
result = operations_on_api().find_violated_company()
return {"Results": result}
# Printing no Violated Companies names
@app.get(api_endpoint.no_violated_company, tags=["Get No Violated Companies"])
async def no_violated_company():
result = operations_on_api().find_no_violated_company()
return {"Results": result}
# Create Excel file
@app.post(api_endpoint.generate_excel_file, tags=["Generate Excel file"])
async def generate_excel_file():
operations_on_api().create_excel_file()
read_excel_file()
return {"Message": "Excel file created"}
scripts/services/main.py deleted 100644 → 0
# Importing Libraries
from fastapi import FastAPI
from scripts.constants.db_connection import db_connect
from scripts.core.handlers.aggregate_operations import violated_companies, no_violated_companies
from scripts.core.handlers.create_excel_file import create_excel
from scripts.core.handlers.upload_and_insert_data import upload_and_insert_into_db
from scripts.utils.mongo_util import create_collection
# Creating FastAPI object
app = FastAPI()
# Root
@app.get("/")
async def root():
return "It's Working!"
# Create Database Connection
db = db_connect()
collection = create_collection(db)
# upload file and insert into db
upload_and_insert_into_db(app, collection)
# Printing Violated Companies names
violated_companies(app, collection)
# Printing no Violated Companies names
no_violated_companies(app, collection)
# Create Excel file
create_excel(app, collection)
scripts/utils/mongo_util.py deleted 100644 → 0
# function for creating collection
def create_collection(db):
# Create Collection
collection_name = "tbl_company"
collection = db[collection_name]
return collection
temp/mongodb_data.xlsx
No preview for this file type
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment