Commit 15d2cba3 by mohammed.shibili

json parsing

parents
# Default ignored files
/shelf/
/workspace.xml
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (jsonParsing)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/jsonParsing.iml" filepath="$PROJECT_DIR$/.idea/jsonParsing.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
from script.servieces.main_call import json_parsing
json_parsing()
[path]
path=script/temp/Task3.xlsx
json_path=script/temp/data.json
pivoted_path=script/temp/pivoted.json
melted_path=script/temp/melted.json
merged_path=script/temp/merged.json
[time_format]
format=%H:%M:%S %d-%m-%y
\ No newline at end of file
{"header": {"label1": "Timestamp", "label2": "kWh", "label3": "kVAh", "label4": "kW", "label5": "kVA", "label6": "current"}, "object": [{"Timestamp": "21:00:00 01-12-19", "kWh": 46.228215767638176, "kVAh": 49.759336099588836, "kW": 183.66897265625, "kVA": 200.53249609375, "current": 281.58292388916016}, {"Timestamp": "21:15:00 01-12-19", "kWh": 45.76348547717498, "kVAh": 50.2282157676309, "kW": 183.46255208333332, "kVA": 200.56794270833333, "current": 281.617431640625}, {"Timestamp": "21:30:00 01-12-19", "kWh": 45.76348547718226, "kVAh": 50.49377593360987, "kW": 184.88955078125, "kVA": 201.79479687499997, "current": 283.0400848388672}, {"Timestamp": "21:45:00 01-12-19", "kWh": 46.552716320351465, "kVAh": 50.48294589788566, "kW": 184.3258984375, "kVA": 201.42360937499998, "current": 282.44676971435547}, {"Timestamp": "22:00:00 01-12-19", "kWh": 46.00458310000249, "kVAh": 50.13828877951164, "kW": 182.37034895833335, "kVA": 199.271421875, "current": 279.42852783203125}, {"Timestamp": "22:15:00 01-12-19", "kWh": 46.110687022897764, "kVAh": 50.63358778625843, "kW": 184.18944140625, "kVA": 201.03991015625, "current": 281.9737777709961}, {"Timestamp": "22:30:00 01-12-19", "kWh": 46.47655815858161, "kVAh": 50.83407914363488, "kW": 186.40050520833333, "kVA": 203.86802604166667, "current": 285.9699300130208}, {"Timestamp": "22:45:00 01-12-19", "kWh": 46.296574167587096, "kVAh": 50.86875390756177, "kW": 185.91884374999998, "kVA": 203.10286718749998, "current": 285.10118103027344}, {"Timestamp": "23:00:00 01-12-19", "kWh": 46.59622562891309, "kVAh": 51.31205402872001, "kW": 185.94254687499998, "kVA": 203.36761458333334, "current": 285.23462931315106}, {"Timestamp": "23:15:00 01-12-19", "kWh": 46.76348547718226, "kVAh": 50.514522821576975, "kW": 185.391484375, "kVA": 202.44480468749998, "current": 283.9910583496094}, {"Timestamp": "18:30:00 02-12-19", "kWh": 46.56323514282849, "kVAh": 50.29440005487413, "kW": 184.82915104166668, "kVA": 202.09330729166666, "current": 283.2255350748698}, {"Timestamp": "18:45:00 02-12-19", "kWh": 45.987603305780794, "kVAh": 50.71900826445926, "kW": 184.39455078125002, "kVA": 201.15467187500002, "current": 281.9291687011719}, {"Timestamp": "19:00:00 02-12-19", "kWh": 46.71440921193425, "kVAh": 50.687471904282575, "kW": 185.46109374999997, "kVA": 202.85015625, "current": 284.17213439941406}, {"Timestamp": "19:15:00 02-12-19", "kWh": 45.83848677928472, "kVAh": 50.32816541953798, "kW": 184.31598958333333, "kVA": 201.36305208333332, "current": 282.21966552734375}, {"Timestamp": "19:30:00 02-12-19", "kWh": 46.59999657076696, "kVAh": 50.61659408114065, "kW": 184.44642578124999, "kVA": 201.67296484375, "current": 282.61204528808594}, {"Timestamp": "19:45:00 02-12-19", "kWh": 46.07635197695345, "kVAh": 50.9892150474916, "kW": 182.8969296875, "kVA": 199.87789062499996, "current": 280.1493377685547}, {"Timestamp": "20:00:00 02-12-19", "kWh": 45.95020746887894, "kVAh": 50.03734439834079, "kW": 182.440359375, "kVA": 199.39240234375, "current": 279.73744201660156}, {"Timestamp": "20:15:00 02-12-19", "kWh": 46.498233942598745, "kVAh": 50.41558931449981, "kW": 184.44230729166668, "kVA": 201.8952552083333, "current": 283.39756266276044}, {"Timestamp": "20:30:00 02-12-19", "kWh": 45.89180755117559, "kVAh": 50.27735674359428, "kW": 183.25395312499998, "kVA": 200.42523828125, "current": 281.13245391845703}, {"Timestamp": "20:45:00 02-12-19", "kWh": 41.18257261411054, "kVAh": 44.87966804979078, "kW": 183.2521640625, "kVA": 200.4773046875, "current": 281.26131439208984}]}
\ No newline at end of file
import configparser
# for mongodb
config = configparser.RawConfigParser()
config.read("conf/data.conf")
path_name = config.get("path", "path")
json_path = config.get("path", "json_path")
pivot_path = config.get("path", "pivoted_path")
melted_path = config.get("path", "melted_path")
merged_path = config.get("path", "merged_path")
date_time_format = config.get("time_format", "format")
import json
from script.config.configuration import json_path, pivot_path, melted_path, merged_path
from script.core.handlers.merge import merge_frame
from script.core.handlers.pivot import to_pivot
# creating json
def to_json(new_dataframe, label_dataframe, melting_data):
header_dict = {'header': dict(zip(label_dataframe.columns, new_dataframe.columns))}
data_dict = {'body': new_dataframe.to_dict(orient='records')}
json_dict = {**header_dict, **data_dict}
# for pivot data
pivoted_data = to_pivot(new_dataframe)
pivoted_dataframe = {'body': pivoted_data.to_dict(orient='records')}
pivot_dict = {**header_dict, **pivoted_dataframe}
try:
with open(json_path, 'w') as f:
json.dump(json_dict, f, indent=4)
except Exception as e:
print("error while parsing ", e)
# pivoted data to json
try:
with open(pivot_path, 'w') as w:
json.dump(pivot_dict, w, indent=4)
except Exception as e:
print("error while parsing of pivot", e)
# for melt
melted_dataframe = {'body': melting_data.to_dict(orient='records')}
melted_dict = {**header_dict, **melted_dataframe}
try:
with open(melted_path, 'w') as w:
json.dump(melted_dict, w, indent=4)
except Exception as e:
print("error while parsing melted data", e)
# merged data
merged_data = merge_frame(new_dataframe)
merged_dataframe = {'body': merged_data.to_dict(orient='records')}
merged_dict = {**header_dict, **merged_dataframe}
try:
with open(merged_path, 'w') as x:
json.dump(merged_dict, x, indent=4)
except Exception as e:
print("error while parsing merged data", e)
def melting_data(new_dataframe):
try:
melted_data = new_dataframe.melt(id_vars=['Timestamp'], value_vars=['kWh'])
return melted_data
except Exception as e:
print("error while melting ", e)
import pandas as pd
def merge_frame(new_dataframe):
first_df = pd.DataFrame(new_dataframe[['Timestamp', 'kWh']])
second_df = pd.DataFrame(new_dataframe[['Timestamp', 'kW']])
try:
merged = pd.merge(first_df, second_df, on='Timestamp', left_index=False, right_index=False)
return merged
except Exception as e:
print("error while merging ", e)
def to_pivot(new_dataframe):
pivoted = new_dataframe.pivot_table(index='Timestamp', columns=[], values=new_dataframe)
return pivoted
from script.config.configuration import date_time_format
def time_format(new_dataframe):
try:
new_dataframe['Timestamp'] = new_dataframe['Timestamp'].dt.strftime(date_time_format)
return new_dataframe
except Exception as e:
print("error while changing time format ", e)
from script.config.configuration import path_name
from script.core.handlers.json import to_json
from script.core.handlers.melt import melting_data
from script.core.handlers.pivot import to_pivot
from script.core.handlers.timeformst import time_format
def json_parsing():
import pandas as pd
dataset = pd.read_excel(path_name)
new_dataframe = pd.DataFrame()
# creating new dataframe by concat
new_dataframe = pd.concat([new_dataframe, dataset.head(10)], ignore_index=True)
new_dataframe = pd.concat([new_dataframe, dataset.tail(10)], ignore_index=True)
# dataframe for setting labels
label_dataframe = pd.DataFrame(
{'label1': '', 'label2': '', 'label3': '', 'label4': '', 'label5': '', 'label6': ''},
index=['column1', 'column2', 'column3', 'column4', 'column5', 'column6', ])
new_dataframe = time_format(new_dataframe)
# call for parsing to json
melt_data = melting_data(new_dataframe)
to_json(new_dataframe, label_dataframe, melt_data)
to_pivot(new_dataframe)
{
"header": {
"label1": "Timestamp",
"label2": "kWh",
"label3": "kVAh",
"label4": "kW",
"label5": "kVA",
"label6": "current"
},
"body": [
{
"Timestamp": "21:00:00 01-12-19",
"kWh": 46.228215767638176,
"kVAh": 49.759336099588836,
"kW": 183.66897265625,
"kVA": 200.53249609375,
"current": 281.58292388916016
},
{
"Timestamp": "21:15:00 01-12-19",
"kWh": 45.76348547717498,
"kVAh": 50.2282157676309,
"kW": 183.46255208333332,
"kVA": 200.56794270833333,
"current": 281.617431640625
},
{
"Timestamp": "21:30:00 01-12-19",
"kWh": 45.76348547718226,
"kVAh": 50.49377593360987,
"kW": 184.88955078125,
"kVA": 201.79479687499997,
"current": 283.0400848388672
},
{
"Timestamp": "21:45:00 01-12-19",
"kWh": 46.552716320351465,
"kVAh": 50.48294589788566,
"kW": 184.3258984375,
"kVA": 201.42360937499998,
"current": 282.44676971435547
},
{
"Timestamp": "22:00:00 01-12-19",
"kWh": 46.00458310000249,
"kVAh": 50.13828877951164,
"kW": 182.37034895833335,
"kVA": 199.271421875,
"current": 279.42852783203125
},
{
"Timestamp": "22:15:00 01-12-19",
"kWh": 46.110687022897764,
"kVAh": 50.63358778625843,
"kW": 184.18944140625,
"kVA": 201.03991015625,
"current": 281.9737777709961
},
{
"Timestamp": "22:30:00 01-12-19",
"kWh": 46.47655815858161,
"kVAh": 50.83407914363488,
"kW": 186.40050520833333,
"kVA": 203.86802604166667,
"current": 285.9699300130208
},
{
"Timestamp": "22:45:00 01-12-19",
"kWh": 46.296574167587096,
"kVAh": 50.86875390756177,
"kW": 185.91884374999998,
"kVA": 203.10286718749998,
"current": 285.10118103027344
},
{
"Timestamp": "23:00:00 01-12-19",
"kWh": 46.59622562891309,
"kVAh": 51.31205402872001,
"kW": 185.94254687499998,
"kVA": 203.36761458333334,
"current": 285.23462931315106
},
{
"Timestamp": "23:15:00 01-12-19",
"kWh": 46.76348547718226,
"kVAh": 50.514522821576975,
"kW": 185.391484375,
"kVA": 202.44480468749998,
"current": 283.9910583496094
},
{
"Timestamp": "18:30:00 02-12-19",
"kWh": 46.56323514282849,
"kVAh": 50.29440005487413,
"kW": 184.82915104166668,
"kVA": 202.09330729166666,
"current": 283.2255350748698
},
{
"Timestamp": "18:45:00 02-12-19",
"kWh": 45.987603305780794,
"kVAh": 50.71900826445926,
"kW": 184.39455078125002,
"kVA": 201.15467187500002,
"current": 281.9291687011719
},
{
"Timestamp": "19:00:00 02-12-19",
"kWh": 46.71440921193425,
"kVAh": 50.687471904282575,
"kW": 185.46109374999997,
"kVA": 202.85015625,
"current": 284.17213439941406
},
{
"Timestamp": "19:15:00 02-12-19",
"kWh": 45.83848677928472,
"kVAh": 50.32816541953798,
"kW": 184.31598958333333,
"kVA": 201.36305208333332,
"current": 282.21966552734375
},
{
"Timestamp": "19:30:00 02-12-19",
"kWh": 46.59999657076696,
"kVAh": 50.61659408114065,
"kW": 184.44642578124999,
"kVA": 201.67296484375,
"current": 282.61204528808594
},
{
"Timestamp": "19:45:00 02-12-19",
"kWh": 46.07635197695345,
"kVAh": 50.9892150474916,
"kW": 182.8969296875,
"kVA": 199.87789062499996,
"current": 280.1493377685547
},
{
"Timestamp": "20:00:00 02-12-19",
"kWh": 45.95020746887894,
"kVAh": 50.03734439834079,
"kW": 182.440359375,
"kVA": 199.39240234375,
"current": 279.73744201660156
},
{
"Timestamp": "20:15:00 02-12-19",
"kWh": 46.498233942598745,
"kVAh": 50.41558931449981,
"kW": 184.44230729166668,
"kVA": 201.8952552083333,
"current": 283.39756266276044
},
{
"Timestamp": "20:30:00 02-12-19",
"kWh": 45.89180755117559,
"kVAh": 50.27735674359428,
"kW": 183.25395312499998,
"kVA": 200.42523828125,
"current": 281.13245391845703
},
{
"Timestamp": "20:45:00 02-12-19",
"kWh": 41.18257261411054,
"kVAh": 44.87966804979078,
"kW": 183.2521640625,
"kVA": 200.4773046875,
"current": 281.26131439208984
}
]
}
\ No newline at end of file
{
"header": {
"label1": "Timestamp",
"label2": "kWh",
"label3": "kVAh",
"label4": "kW",
"label5": "kVA",
"label6": "current"
},
"body": [
{
"Timestamp": "21:00:00 01-12-19",
"variable": "kWh",
"value": 46.228215767638176
},
{
"Timestamp": "21:15:00 01-12-19",
"variable": "kWh",
"value": 45.76348547717498
},
{
"Timestamp": "21:30:00 01-12-19",
"variable": "kWh",
"value": 45.76348547718226
},
{
"Timestamp": "21:45:00 01-12-19",
"variable": "kWh",
"value": 46.552716320351465
},
{
"Timestamp": "22:00:00 01-12-19",
"variable": "kWh",
"value": 46.00458310000249
},
{
"Timestamp": "22:15:00 01-12-19",
"variable": "kWh",
"value": 46.110687022897764
},
{
"Timestamp": "22:30:00 01-12-19",
"variable": "kWh",
"value": 46.47655815858161
},
{
"Timestamp": "22:45:00 01-12-19",
"variable": "kWh",
"value": 46.296574167587096
},
{
"Timestamp": "23:00:00 01-12-19",
"variable": "kWh",
"value": 46.59622562891309
},
{
"Timestamp": "23:15:00 01-12-19",
"variable": "kWh",
"value": 46.76348547718226
},
{
"Timestamp": "18:30:00 02-12-19",
"variable": "kWh",
"value": 46.56323514282849
},
{
"Timestamp": "18:45:00 02-12-19",
"variable": "kWh",
"value": 45.987603305780794
},
{
"Timestamp": "19:00:00 02-12-19",
"variable": "kWh",
"value": 46.71440921193425
},
{
"Timestamp": "19:15:00 02-12-19",
"variable": "kWh",
"value": 45.83848677928472
},
{
"Timestamp": "19:30:00 02-12-19",
"variable": "kWh",
"value": 46.59999657076696
},
{
"Timestamp": "19:45:00 02-12-19",
"variable": "kWh",
"value": 46.07635197695345
},
{
"Timestamp": "20:00:00 02-12-19",
"variable": "kWh",
"value": 45.95020746887894
},
{
"Timestamp": "20:15:00 02-12-19",
"variable": "kWh",
"value": 46.498233942598745
},
{
"Timestamp": "20:30:00 02-12-19",
"variable": "kWh",
"value": 45.89180755117559
},
{
"Timestamp": "20:45:00 02-12-19",
"variable": "kWh",
"value": 41.18257261411054
}
]
}
\ No newline at end of file
{
"header": {
"label1": "Timestamp",
"label2": "kWh",
"label3": "kVAh",
"label4": "kW",
"label5": "kVA",
"label6": "current"
},
"body": [
{
"Timestamp": "21:00:00 01-12-19",
"kWh": 46.228215767638176,
"kW": 183.66897265625
},
{
"Timestamp": "21:15:00 01-12-19",
"kWh": 45.76348547717498,
"kW": 183.46255208333332
},
{
"Timestamp": "21:30:00 01-12-19",
"kWh": 45.76348547718226,
"kW": 184.88955078125
},
{
"Timestamp": "21:45:00 01-12-19",
"kWh": 46.552716320351465,
"kW": 184.3258984375
},
{
"Timestamp": "22:00:00 01-12-19",
"kWh": 46.00458310000249,
"kW": 182.37034895833335
},
{
"Timestamp": "22:15:00 01-12-19",
"kWh": 46.110687022897764,
"kW": 184.18944140625
},
{
"Timestamp": "22:30:00 01-12-19",
"kWh": 46.47655815858161,
"kW": 186.40050520833333
},
{
"Timestamp": "22:45:00 01-12-19",
"kWh": 46.296574167587096,
"kW": 185.91884374999998
},
{
"Timestamp": "23:00:00 01-12-19",
"kWh": 46.59622562891309,
"kW": 185.94254687499998
},
{
"Timestamp": "23:15:00 01-12-19",
"kWh": 46.76348547718226,
"kW": 185.391484375
},
{
"Timestamp": "18:30:00 02-12-19",
"kWh": 46.56323514282849,
"kW": 184.82915104166668
},
{
"Timestamp": "18:45:00 02-12-19",
"kWh": 45.987603305780794,
"kW": 184.39455078125002
},
{
"Timestamp": "19:00:00 02-12-19",
"kWh": 46.71440921193425,
"kW": 185.46109374999997
},
{
"Timestamp": "19:15:00 02-12-19",
"kWh": 45.83848677928472,
"kW": 184.31598958333333
},
{
"Timestamp": "19:30:00 02-12-19",
"kWh": 46.59999657076696,
"kW": 184.44642578124999
},
{
"Timestamp": "19:45:00 02-12-19",
"kWh": 46.07635197695345,
"kW": 182.8969296875
},
{
"Timestamp": "20:00:00 02-12-19",
"kWh": 45.95020746887894,
"kW": 182.440359375
},
{
"Timestamp": "20:15:00 02-12-19",
"kWh": 46.498233942598745,
"kW": 184.44230729166668
},
{
"Timestamp": "20:30:00 02-12-19",
"kWh": 45.89180755117559,
"kW": 183.25395312499998
},
{
"Timestamp": "20:45:00 02-12-19",
"kWh": 41.18257261411054,
"kW": 183.2521640625
}
]
}
\ No newline at end of file
{
"header": {
"label1": "Timestamp",
"label2": "kWh",
"label3": "kVAh",
"label4": "kW",
"label5": "kVA",
"label6": "current"
},
"body": [
{
"current": 283.2255350748698,
"kVA": 202.09330729166666,
"kVAh": 50.29440005487413,
"kW": 184.82915104166668,
"kWh": 46.56323514282849
},
{
"current": 281.9291687011719,
"kVA": 201.15467187500002,
"kVAh": 50.71900826445926,
"kW": 184.39455078125002,
"kWh": 45.987603305780794
},
{
"current": 284.17213439941406,
"kVA": 202.85015625,
"kVAh": 50.687471904282575,
"kW": 185.46109374999997,
"kWh": 46.71440921193425
},
{
"current": 282.21966552734375,
"kVA": 201.36305208333332,
"kVAh": 50.32816541953798,
"kW": 184.31598958333333,
"kWh": 45.83848677928472
},
{
"current": 282.61204528808594,
"kVA": 201.67296484375,
"kVAh": 50.61659408114065,
"kW": 184.44642578124999,
"kWh": 46.59999657076696
},
{
"current": 280.1493377685547,
"kVA": 199.87789062499996,
"kVAh": 50.9892150474916,
"kW": 182.8969296875,
"kWh": 46.07635197695345
},
{
"current": 279.73744201660156,
"kVA": 199.39240234375,
"kVAh": 50.03734439834079,
"kW": 182.440359375,
"kWh": 45.95020746887894
},
{
"current": 283.39756266276044,
"kVA": 201.8952552083333,
"kVAh": 50.41558931449981,
"kW": 184.44230729166668,
"kWh": 46.498233942598745
},
{
"current": 281.13245391845703,
"kVA": 200.42523828125,
"kVAh": 50.27735674359428,
"kW": 183.25395312499998,
"kWh": 45.89180755117559
},
{
"current": 281.26131439208984,
"kVA": 200.4773046875,
"kVAh": 44.87966804979078,
"kW": 183.2521640625,
"kWh": 41.18257261411054
},
{
"current": 281.58292388916016,
"kVA": 200.53249609375,
"kVAh": 49.759336099588836,
"kW": 183.66897265625,
"kWh": 46.228215767638176
},
{
"current": 281.617431640625,
"kVA": 200.56794270833333,
"kVAh": 50.2282157676309,
"kW": 183.46255208333332,
"kWh": 45.76348547717498
},
{
"current": 283.0400848388672,
"kVA": 201.79479687499997,
"kVAh": 50.49377593360987,
"kW": 184.88955078125,
"kWh": 45.76348547718226
},
{
"current": 282.44676971435547,
"kVA": 201.42360937499998,
"kVAh": 50.48294589788566,
"kW": 184.3258984375,
"kWh": 46.552716320351465
},
{
"current": 279.42852783203125,
"kVA": 199.271421875,
"kVAh": 50.13828877951164,
"kW": 182.37034895833335,
"kWh": 46.00458310000249
},
{
"current": 281.9737777709961,
"kVA": 201.03991015625,
"kVAh": 50.63358778625843,
"kW": 184.18944140625,
"kWh": 46.110687022897764
},
{
"current": 285.9699300130208,
"kVA": 203.86802604166667,
"kVAh": 50.83407914363488,
"kW": 186.40050520833333,
"kWh": 46.47655815858161
},
{
"current": 285.10118103027344,
"kVA": 203.10286718749998,
"kVAh": 50.86875390756177,
"kW": 185.91884374999998,
"kWh": 46.296574167587096
},
{
"current": 285.23462931315106,
"kVA": 203.36761458333334,
"kVAh": 51.31205402872001,
"kW": 185.94254687499998,
"kWh": 46.59622562891309
},
{
"current": 283.9910583496094,
"kVA": 202.44480468749998,
"kVAh": 50.514522821576975,
"kW": 185.391484375,
"kWh": 46.76348547718226
}
]
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment