staudtyves/ConsultancyProject_2_ETL
1
0
Fork 0
forked from stoffelmauro/ConsultancyProject_2_ETL
Code Pull Requests Activity

Pakete polars und fastAPI hinzugefügt, Database Klasse verbessert, FastAPI route erstellt.

Browse Source
This commit is contained in:
Giò Diani 2024-11-04 21:36:34 +01:00
parent 219ad57a35
commit e2734a3db5
7 changed files with 6362 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6460
pixi.lock generated
View File

File diff suppressed because it is too large Load Diff

3
pyproject.toml
View File

@ -25,3 +25,6 @@ pandas = ">=2.2.3,<3"
plotly = ">=5.24.1,<6"
duckdb = ">=1.1.2,<2"
python-dotenv = ">=1.0.1,<2"
fastapi = ">=0.115.4,<0.116"
polars = ">=0.20.26,<2"
pyarrow = ">=18.0.0,<19"

22
src/dashboard/main.py Normal file
View File

@ -0,0 +1,22 @@
from typing import Union
import polars as pl
from fastapi import FastAPI, Response
import data
d = data.load()
app = FastAPI()
@app.get("/")
def read_root():
return {"Hello": "World"}
@app.get("/items/{item_id}")
def read_item(item_id: int):
ext = d.extractions_for(item_id).pl()
out = ext.with_columns(pl.col("calendar").str.extract_all(r"([0-9]{4}-[0-9]{2}-[0-9]{2})|[0-2]").alias("calendar_data"))
out = out.drop(['calendar', 'property_id'])
return Response(content=out.write_json(), media_type="application/json")

5
src/data/__init__.py
View File

@ -4,7 +4,10 @@ from dotenv import load_dotenv
from data import database
load_dotenv(dotenv_path='../.env')
dirname = os.path.dirname(__file__)
envfile = os.path.join(dirname, '../.env')
load_dotenv(dotenv_path=envfile)
def load():
return database.Database(path=os.getenv('DATABASE'))

51
src/data/database.py
View File

@ -1,3 +1,5 @@
from threading import Thread, current_thread
import duckdb
@ -18,7 +20,8 @@ class Database:
).fetchone()
def __init__(self, path):
self.connection = duckdb.connect(database = path)
duckdb_connection = duckdb.connect(database = path, read_only=True)
self.connection = duckdb_connection.cursor()
# Install spatial extension if not already installed
spatial_installed = self.check_duckdb_extensions(extension='spatial')
@ -26,6 +29,7 @@ class Database:
self.connection.sql("INSTALL spatial")
def db_overview(self):
return self.connection.sql("DESCRIBE;").show()
@ -49,7 +53,7 @@ class Database:
consultancy_d.properties
GROUP BY
date;
""").show()
""")
def properties_per_region(self):
return self.connection.sql("""
@ -65,7 +69,7 @@ class Database:
GROUP BY
properties.seed_id,
regions.name
""").show()
""")
def properties_unreachable(self):
return self.connection.sql("""
@ -178,5 +182,46 @@ class Database:
""")
def properties_exceptions(self):
return self.connection.sql("""
SELECT
JSON_EXTRACT(exception, '$.status') AS exception_status,
COUNT(JSON_EXTRACT(exception, '$.status')) AS exception_count
FROM
consultancy_d.exceptions
WHERE
type != 'property'
GROUP BY
JSON_EXTRACT(exception, '$.status')
""")
def extractions(self):
return self.connection.sql("""
SELECT
JSON_EXTRACT(exception, '$.status') AS exception_status,
COUNT(JSON_EXTRACT(exception, '$.status')) AS exception_count
FROM
consultancy_d.extractions
WHERE
type != 'property'
GROUP BY
extractions.date
""")
def extractions_for(self, property_id):
return self.connection.sql(f"""
SELECT
JSON_EXTRACT(body, '$.content.days') as calendar,
property_id,
created_at
FROM
consultancy_d.extractions
WHERE
type == 'calendar' AND
property_id = {property_id}
ORDER BY
property_id
""")

66
src/gio/index.html Normal file
View File

@ -0,0 +1,66 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Document</title>
<script src=" https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js "></script>
</head>
<body>
<div style="width: 100%; height: 800px;" id="main"></div>
<script type="text/javascript">
var chartDom = document.getElementById('main');
var myChart = echarts.init(chartDom);
var option;
function getVirtualData(year) {
const date = +echarts.time.parse(year + '-01-01');
const end = +echarts.time.parse(+year + 1 + '-01-01');
const dayTime = 3600 * 24 * 1000;
const data = [];
for (let time = date; time < end; time += dayTime) {
data.push([
echarts.time.format(time, '{yyyy}-{MM}-{dd}', false),
Math.floor(Math.random() * 1000)
]);
}
return data;
}
console.log(getVirtualData('2017'))
option = {
tooltip: {
position: 'top'
},
visualMap: {
min: 0,
max: 1000,
calculable: true,
orient: 'horizontal',
left: 'center',
top: 'top'
},
calendar: [
{
range: '2024',
cellSize: ['auto', 20]
},
],
series: [
{
type: 'heatmap',
coordinateSystem: 'calendar',
calendarIndex: 0,
data: getVirtualData('2017')
},
]
};
option && myChart.setOption(option);
</script>
</body>
</html>

14
src/gio/test_duckdb.py
View File

@ -1,4 +1,16 @@
import polars as pl
import data
inst = data.load()
inst.properties_distance().show()
test = inst.extractions_for(1).pl()
out = test.with_columns(
pl.col("calendar").str.extract_all(r"([0-9]{4}-[0-9]{2}-[0-9]{2})|[0-2]").alias("extracted_cal"),
)
out = out.drop(['calendar', 'property_id'])
print(out.to_dict(as_series=True))