Commit 1e967cbf by arun.uday

v1

parents
# Default ignored files
/shelf/
/workspace.xml
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (task5)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/task5.iml" filepath="$PROJECT_DIR$/.idea/task5.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
# project to access Json and pandas
from scripts.constants.app_constants import exception_msg
from scripts.services.access_operations import access_op
print("Json and Pandas")
try:
access_op()
except Exception as e:
print(f'{exception_msg} {e}')
else:
print("Program successful")
[file_path]
excel_path = scripts/external_files/task 3.xlsx
json_time = scripts/json_output/data_time_json.json
json_pivot = scripts/json_output/data_pivot_json.json
json_melt= scripts/json_output/data_melt_json.json
json_merge= scripts/json_output/data_merge_json.json
[Date_format]
date_format = %d-%m-%y %S-%M-%H
# Reading the conf file
import configparser
# config object
config = configparser.RawConfigParser()
config.read("conf/application.conf")
# reading paths from conf file
path_excel = config.get("file_path", "excel_path")
path_time = config.get("file_path", "json_time")
path_pivot = config.get("file_path", "json_pivot")
path_melt = config.get("file_path", "json_melt")
path_merge = config.get("file_path", "json_merge")
# reading date format from conf file
date_format = config.get("Date_format", "date_format")
# Exception message
exception_msg = "Exception occurred"
# change the time format
from scripts.config.application_config import date_format
def time_change(file):
file_copy = file
# creating the changed time format
file_copy['Timestamp'] = file_copy['Timestamp'].dt.strftime(date_format)
# for storing the pivoted frame to dictionary
dict_cols = {}
for i in range(0, len(file_copy.columns)):
key_dict = "label" + str(i)
val_dict = file_copy.columns[i]
dict_cols.update({key_dict: val_dict})
# storing the header and body to json
head_cols = {"header": dict_cols}
body_vals = {"body": file_copy.to_dict(orient="records")}
dict_new = {**head_cols, **body_vals}
return dict_new
# Reading the csv file
import pandas as pd
from scripts.config.application_config import path_excel
# reading the file from the path
def read_from_path():
data_csv = pd.read_excel(path_excel, index_col=None)
return data_csv
# concatenating first 10 and last 10
import pandas as pd
# concatenating the head and tail part of the data frame
def data_concat_excel(file):
concatenate_data = pd.DataFrame()
concatenate_data = pd.concat([concatenate_data, file.head(10)], ignore_index=True)
concatenate_data = pd.concat([concatenate_data, file.tail(10)], ignore_index=True)
return concatenate_data
# printing dataframe as json
import json
def print_json(path, file):
result = file
# outputting the dictionary in to a json file
with open(path, "w") as f:
json.dump(result, f, indent=4)
# melting the dataframe
def melt_frame(file):
file_copy = file
# creating the melted dataframe melted_file
melted_file = file_copy.melt(id_vars=['Timestamp'], value_vars=['kWh'])
# for storing the melted frame to dictionary
dict_cols = {}
for i in range(0, len(melted_file.columns)):
key_dict = "label" + str(i)
val_dict = melted_file.columns[i]
dict_cols.update({key_dict: val_dict})
# storing the header and body to json
head_cols = {"header": dict_cols}
body_vals = {"body": melted_file.to_dict(orient="records")}
dict_new = {**head_cols, **body_vals}
return dict_new
# merging the dataframe
import pandas as pd
def merge_frame(file):
file_copy = file
# creating two dataframes for the merge
file_frame1 = file_copy[['Timestamp', 'kW']]
file_frame2 = file_copy[['Timestamp', 'kVA']]
# creating the merged dataframe merged_file
merged_file = pd.merge(file_frame1, file_frame2, on='Timestamp')
# for storing the merged frame to dictionary
dict_cols = {}
for i in range(0, len(merged_file.columns)):
key_dict = "label" + str(i)
val_dict = merged_file.columns[i]
dict_cols.update({key_dict: val_dict})
# storing the header and body to json
head_cols = {"header": dict_cols}
body_vals = {"body": merged_file.to_dict(orient="records")}
dict_new = {**head_cols, **body_vals}
return dict_new
# pivot the dataframe
def pivot_frame(file):
file_copy = file
# pivoting the data
pivoted_d = file_copy.pivot_table(index='Timestamp', columns=[],
values=file_copy)
# for storing the pivoted frame to dictionary
dict_cols = {}
for i in range(0, len(pivoted_d.columns)):
key_dict = "label"+str(i)
val_dict = pivoted_d.columns[i]
dict_cols.update({key_dict: val_dict})
# storing the header and body to json
head_cols = {"header": dict_cols}
body_vals = {"body": pivoted_d.to_dict(orient="records")}
dict_new = {**head_cols, **body_vals}
return dict_new
{
"header": {
"label0": "Timestamp",
"label1": "variable",
"label2": "value"
},
"body": [
{
"Timestamp": "01-12-19 00-00-21",
"variable": "kWh",
"value": 46.228215767638176
},
{
"Timestamp": "01-12-19 00-15-21",
"variable": "kWh",
"value": 45.76348547717498
},
{
"Timestamp": "01-12-19 00-30-21",
"variable": "kWh",
"value": 45.76348547718226
},
{
"Timestamp": "01-12-19 00-45-21",
"variable": "kWh",
"value": 46.552716320351465
},
{
"Timestamp": "01-12-19 00-00-22",
"variable": "kWh",
"value": 46.00458310000249
},
{
"Timestamp": "01-12-19 00-15-22",
"variable": "kWh",
"value": 46.110687022897764
},
{
"Timestamp": "01-12-19 00-30-22",
"variable": "kWh",
"value": 46.47655815858161
},
{
"Timestamp": "01-12-19 00-45-22",
"variable": "kWh",
"value": 46.296574167587096
},
{
"Timestamp": "01-12-19 00-00-23",
"variable": "kWh",
"value": 46.59622562891309
},
{
"Timestamp": "01-12-19 00-15-23",
"variable": "kWh",
"value": 46.76348547718226
},
{
"Timestamp": "02-12-19 00-15-19",
"variable": "kWh",
"value": 45.83848677928472
},
{
"Timestamp": "02-12-19 00-30-19",
"variable": "kWh",
"value": 46.59999657076696
},
{
"Timestamp": "02-12-19 00-45-19",
"variable": "kWh",
"value": 46.07635197695345
},
{
"Timestamp": "02-12-19 00-00-20",
"variable": "kWh",
"value": 45.95020746887894
},
{
"Timestamp": "02-12-19 00-15-20",
"variable": "kWh",
"value": 46.498233942598745
},
{
"Timestamp": "02-12-19 00-30-20",
"variable": "kWh",
"value": 45.89180755117559
},
{
"Timestamp": "02-12-19 00-45-20",
"variable": "kWh",
"value": 41.18257261411054
},
{
"Timestamp": "02-12-19 00-45-21",
"variable": "kWh",
"value": 45.1825726141105
},
{
"Timestamp": "02-12-19 00-45-22",
"variable": "kWh",
"value": 46.1825726141105
},
{
"Timestamp": "02-12-19 00-45-23",
"variable": "kWh",
"value": 45.1825726141105
}
]
}
\ No newline at end of file
{
"header": {
"label0": "Timestamp",
"label1": "kW",
"label2": "kVA"
},
"body": [
{
"Timestamp": "01-12-19 00-00-21",
"kW": 183.66897265625,
"kVA": 200.53249609375
},
{
"Timestamp": "01-12-19 00-15-21",
"kW": 183.46255208333332,
"kVA": 200.56794270833333
},
{
"Timestamp": "01-12-19 00-30-21",
"kW": 184.88955078125,
"kVA": 201.79479687499997
},
{
"Timestamp": "01-12-19 00-45-21",
"kW": 184.3258984375,
"kVA": 201.42360937499998
},
{
"Timestamp": "01-12-19 00-00-22",
"kW": 182.37034895833335,
"kVA": 199.271421875
},
{
"Timestamp": "01-12-19 00-15-22",
"kW": 184.18944140625,
"kVA": 201.03991015625
},
{
"Timestamp": "01-12-19 00-30-22",
"kW": 186.40050520833333,
"kVA": 203.86802604166667
},
{
"Timestamp": "01-12-19 00-45-22",
"kW": 185.91884374999998,
"kVA": 203.10286718749998
},
{
"Timestamp": "01-12-19 00-00-23",
"kW": 185.94254687499998,
"kVA": 203.36761458333334
},
{
"Timestamp": "01-12-19 00-15-23",
"kW": 185.391484375,
"kVA": 202.44480468749998
},
{
"Timestamp": "02-12-19 00-15-19",
"kW": 184.31598958333333,
"kVA": 201.36305208333332
},
{
"Timestamp": "02-12-19 00-30-19",
"kW": 184.44642578124999,
"kVA": 201.67296484375
},
{
"Timestamp": "02-12-19 00-45-19",
"kW": 182.8969296875,
"kVA": 199.87789062499996
},
{
"Timestamp": "02-12-19 00-00-20",
"kW": 182.440359375,
"kVA": 199.39240234375
},
{
"Timestamp": "02-12-19 00-15-20",
"kW": 184.44230729166668,
"kVA": 201.8952552083333
},
{
"Timestamp": "02-12-19 00-30-20",
"kW": 183.25395312499998,
"kVA": 200.42523828125
},
{
"Timestamp": "02-12-19 00-45-20",
"kW": 185.74748828125,
"kVA": 200.4773046875
},
{
"Timestamp": "02-12-19 00-45-21",
"kW": 186.4073828125,
"kVA": 203.51410546875
},
{
"Timestamp": "02-12-19 00-45-22",
"kW": 183.2521640625,
"kVA": 200.68902734375
},
{
"Timestamp": "02-12-19 00-45-23",
"kW": 184.11314062499997,
"kVA": 203.18090625
}
]
}
\ No newline at end of file
{
"header": {
"label0": "current",
"label1": "kVA",
"label2": "kVAh",
"label3": "kW",
"label4": "kWh"
},
"body": [
{
"current": 281.58292388916016,
"kVA": 200.53249609375,
"kVAh": 49.759336099588836,
"kW": 183.66897265625,
"kWh": 46.228215767638176
},
{
"current": 279.42852783203125,
"kVA": 199.271421875,
"kVAh": 50.13828877951164,
"kW": 182.37034895833335,
"kWh": 46.00458310000249
},
{
"current": 285.23462931315106,
"kVA": 203.36761458333334,
"kVAh": 51.31205402872001,
"kW": 185.94254687499998,
"kWh": 46.59622562891309
},
{
"current": 281.617431640625,
"kVA": 200.56794270833333,
"kVAh": 50.2282157676309,
"kW": 183.46255208333332,
"kWh": 45.76348547717498
},
{
"current": 281.9737777709961,
"kVA": 201.03991015625,
"kVAh": 50.63358778625843,
"kW": 184.18944140625,
"kWh": 46.110687022897764
},
{
"current": 283.9910583496094,
"kVA": 202.44480468749998,
"kVAh": 50.514522821576975,
"kW": 185.391484375,
"kWh": 46.76348547718226
},
{
"current": 283.0400848388672,
"kVA": 201.79479687499997,
"kVAh": 50.49377593360987,
"kW": 184.88955078125,
"kWh": 45.76348547718226
},
{
"current": 285.9699300130208,
"kVA": 203.86802604166667,
"kVAh": 50.83407914363488,
"kW": 186.40050520833333,
"kWh": 46.47655815858161
},
{
"current": 282.44676971435547,
"kVA": 201.42360937499998,
"kVAh": 50.48294589788566,
"kW": 184.3258984375,
"kWh": 46.552716320351465
},
{
"current": 285.10118103027344,
"kVA": 203.10286718749998,
"kVAh": 50.86875390756177,
"kW": 185.91884374999998,
"kWh": 46.296574167587096
},
{
"current": 279.73744201660156,
"kVA": 199.39240234375,
"kVAh": 50.03734439834079,
"kW": 182.440359375,
"kWh": 45.95020746887894
},
{
"current": 282.21966552734375,
"kVA": 201.36305208333332,
"kVAh": 50.32816541953798,
"kW": 184.31598958333333,
"kWh": 45.83848677928472
},
{
"current": 283.39756266276044,
"kVA": 201.8952552083333,
"kVAh": 50.41558931449981,
"kW": 184.44230729166668,
"kWh": 46.498233942598745
},
{
"current": 282.61204528808594,
"kVA": 201.67296484375,
"kVAh": 50.61659408114065,
"kW": 184.44642578124999,
"kWh": 46.59999657076696
},
{
"current": 281.13245391845703,
"kVA": 200.42523828125,
"kVAh": 50.27735674359428,
"kW": 183.25395312499998,
"kWh": 45.89180755117559
},
{
"current": 280.1493377685547,
"kVA": 199.87789062499996,
"kVAh": 50.9892150474916,
"kW": 182.8969296875,
"kWh": 46.07635197695345
},
{
"current": 281.26131439208984,
"kVA": 200.4773046875,
"kVAh": 44.87966804979078,
"kW": 185.74748828125,
"kWh": 41.18257261411054
},
{
"current": 284.3976135253906,
"kVA": 203.51410546875,
"kVAh": 51.1106614999444,
"kW": 186.4073828125,
"kWh": 45.1825726141105
},
{
"current": 284.9558639526367,
"kVA": 200.68902734375,
"kVAh": 50.99694439139421,
"kW": 183.2521640625,
"kWh": 46.1825726141105
},
{
"current": 285.91526794433594,
"kVA": 203.18090625,
"kVAh": 51.14462809917313,
"kW": 184.11314062499997,
"kWh": 45.1825726141105
}
]
}
\ No newline at end of file
{
"header": {
"label0": "Timestamp",
"label1": "kWh",
"label2": "kVAh",
"label3": "kW",
"label4": "kVA",
"label5": "current"
},
"body": [
{
"Timestamp": "01-12-19 00-00-21",
"kWh": 46.228215767638176,
"kVAh": 49.759336099588836,
"kW": 183.66897265625,
"kVA": 200.53249609375,
"current": 281.58292388916016
},
{
"Timestamp": "01-12-19 00-15-21",
"kWh": 45.76348547717498,
"kVAh": 50.2282157676309,
"kW": 183.46255208333332,
"kVA": 200.56794270833333,
"current": 281.617431640625
},
{
"Timestamp": "01-12-19 00-30-21",
"kWh": 45.76348547718226,
"kVAh": 50.49377593360987,
"kW": 184.88955078125,
"kVA": 201.79479687499997,
"current": 283.0400848388672
},
{
"Timestamp": "01-12-19 00-45-21",
"kWh": 46.552716320351465,
"kVAh": 50.48294589788566,
"kW": 184.3258984375,
"kVA": 201.42360937499998,
"current": 282.44676971435547
},
{
"Timestamp": "01-12-19 00-00-22",
"kWh": 46.00458310000249,
"kVAh": 50.13828877951164,
"kW": 182.37034895833335,
"kVA": 199.271421875,
"current": 279.42852783203125
},
{
"Timestamp": "01-12-19 00-15-22",
"kWh": 46.110687022897764,
"kVAh": 50.63358778625843,
"kW": 184.18944140625,
"kVA": 201.03991015625,
"current": 281.9737777709961
},
{
"Timestamp": "01-12-19 00-30-22",
"kWh": 46.47655815858161,
"kVAh": 50.83407914363488,
"kW": 186.40050520833333,
"kVA": 203.86802604166667,
"current": 285.9699300130208
},
{
"Timestamp": "01-12-19 00-45-22",
"kWh": 46.296574167587096,
"kVAh": 50.86875390756177,
"kW": 185.91884374999998,
"kVA": 203.10286718749998,
"current": 285.10118103027344
},
{
"Timestamp": "01-12-19 00-00-23",
"kWh": 46.59622562891309,
"kVAh": 51.31205402872001,
"kW": 185.94254687499998,
"kVA": 203.36761458333334,
"current": 285.23462931315106
},
{
"Timestamp": "01-12-19 00-15-23",
"kWh": 46.76348547718226,
"kVAh": 50.514522821576975,
"kW": 185.391484375,
"kVA": 202.44480468749998,
"current": 283.9910583496094
},
{
"Timestamp": "02-12-19 00-15-19",
"kWh": 45.83848677928472,
"kVAh": 50.32816541953798,
"kW": 184.31598958333333,
"kVA": 201.36305208333332,
"current": 282.21966552734375
},
{
"Timestamp": "02-12-19 00-30-19",
"kWh": 46.59999657076696,
"kVAh": 50.61659408114065,
"kW": 184.44642578124999,
"kVA": 201.67296484375,
"current": 282.61204528808594
},
{
"Timestamp": "02-12-19 00-45-19",
"kWh": 46.07635197695345,
"kVAh": 50.9892150474916,
"kW": 182.8969296875,
"kVA": 199.87789062499996,
"current": 280.1493377685547
},
{
"Timestamp": "02-12-19 00-00-20",
"kWh": 45.95020746887894,
"kVAh": 50.03734439834079,
"kW": 182.440359375,
"kVA": 199.39240234375,
"current": 279.73744201660156
},
{
"Timestamp": "02-12-19 00-15-20",
"kWh": 46.498233942598745,
"kVAh": 50.41558931449981,
"kW": 184.44230729166668,
"kVA": 201.8952552083333,
"current": 283.39756266276044
},
{
"Timestamp": "02-12-19 00-30-20",
"kWh": 45.89180755117559,
"kVAh": 50.27735674359428,
"kW": 183.25395312499998,
"kVA": 200.42523828125,
"current": 281.13245391845703
},
{
"Timestamp": "02-12-19 00-45-20",
"kWh": 41.18257261411054,
"kVAh": 44.87966804979078,
"kW": 185.74748828125,
"kVA": 200.4773046875,
"current": 281.26131439208984
},
{
"Timestamp": "02-12-19 00-45-21",
"kWh": 45.1825726141105,
"kVAh": 51.1106614999444,
"kW": 186.4073828125,
"kVA": 203.51410546875,
"current": 284.3976135253906
},
{
"Timestamp": "02-12-19 00-45-22",
"kWh": 46.1825726141105,
"kVAh": 50.99694439139421,
"kW": 183.2521640625,
"kVA": 200.68902734375,
"current": 284.9558639526367
},
{
"Timestamp": "02-12-19 00-45-23",
"kWh": 45.1825726141105,
"kVAh": 51.14462809917313,
"kW": 184.11314062499997,
"kVA": 203.18090625,
"current": 285.91526794433594
}
]
}
\ No newline at end of file
# project to access Json and pandas
from scripts.config.application_config import path_time, path_pivot, path_melt, path_merge
from scripts.constants.app_constants import exception_msg
from scripts.services.app_class_operations import JsonPandas
def access_op():
try:
json_pandas = JsonPandas()
print("Reading the Json file....")
file_get = json_pandas.read_excel()
file_concat = json_pandas.concat_data(file_get)
print("Changing the time format")
time_changed = json_pandas.to_time_change(file_concat)
json_pandas.create_json(path_time, time_changed)
print("Pivoting the dataframe")
pivot_change = json_pandas.pivot_data(file_concat)
json_pandas.create_json(path_pivot, pivot_change)
print("Melting the dataframe")
melt_change = json_pandas.melt_data(file_concat)
json_pandas.create_json(path_melt, melt_change)
print("Merging the dataframe")
merge_change = json_pandas.merge_data(file_concat)
json_pandas.create_json(path_merge, merge_change)
except Exception as e:
print(f'{exception_msg} {e}')
else:
print("Access complete")
# class for accessing services
from scripts.constants.app_constants import exception_msg
from scripts.core.handlers.change_time_data import time_change
from scripts.core.handlers.csv_read_file import read_from_path
from scripts.core.handlers.data_concat import data_concat_excel
from scripts.core.handlers.json_output_fn import print_json
from scripts.core.handlers.melted_data_frames import melt_frame
from scripts.core.handlers.merge_data_frames import merge_frame
from scripts.core.handlers.pivoted_data_frames import pivot_frame
class JsonPandas:
# read the excel file
@staticmethod
def read_excel():
try:
file_read = read_from_path()
except Exception as e:
print(f'{exception_msg} {e}')
else:
return file_read
# creating the json
@staticmethod
def create_json(path, file):
try:
print_json(path, file)
except Exception as e:
print(f'{exception_msg} {e}')
else:
print("Json file created\n")
# concatenating the data
@staticmethod
def concat_data(file):
try:
concatenated = data_concat_excel(file)
except Exception as e:
print(f'{exception_msg} {e}')
else:
return concatenated
@staticmethod
def to_time_change(file):
try:
time_changed = time_change(file)
except Exception as e:
print(f'{exception_msg} {e}')
else:
return time_changed
# using the pivot function
@staticmethod
def pivot_data(file):
try:
data_pivoted = pivot_frame(file)
except Exception as e:
print(f'{exception_msg}{e}')
else:
return data_pivoted
# using the melt function
@staticmethod
def melt_data(file):
try:
data_melted = melt_frame(file)
except Exception as e:
print(f'{exception_msg} {e}')
else:
return data_melted
# using merge function
@staticmethod
def merge_data(file):
try:
data_merge = merge_frame(file)
except Exception as e:
print(f'{exception_msg} {e}')
else:
return data_merge
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment