stoffelmauro/ConsultancyProject_2_ETL
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Further dashboard development.

Browse Source
This commit is contained in:
Giò Diani 2024-12-18 19:52:06 +01:00
parent 1574edea88
commit a8543d619f
10 changed files with 209 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dashboard/app/Api.php
View File

@ -30,9 +30,24 @@ class Api
return self::get('/region/properties');
}
public static function propertiesGrowth()
{
return self::get('/properties/growth');
}
public static function propertiesGeo()
{
return self::get('/properties/geo');
}
public static function propertyExtractions(int $id)
{
return self::get("/properties/extractions/{$id}");
return self::get("/property/{$id}/extractions");
}
public static function propertyBase(int $id): mixed
{
return self::get("/property/{$id}/base");
}

12
dashboard/resources/css/app.css
View File

@ -46,6 +46,14 @@ h1, h2, h3, h4, h5, h6 {
text-wrap: balance;
}
dt{
font-weight: 600;
}
dd + dt{
margin-top: .2em;
}
/*
9. Create a root stacking context
*/
@ -53,6 +61,10 @@ h1, h2, h3, h4, h5, h6 {
isolation: isolate;
}
nav>ul{
list-style: none;
}
body>header{
position: fixed;

274
dashboard/resources/js/app.js
View File

File diff suppressed because one or more lines are too long

7
dashboard/resources/views/base.blade.php
View File

@ -10,6 +10,13 @@
<body>
<header>
<span>Dashboard</span>
<nav>
<ul>
<li>
<a href="/">Start</a>
</li>
</ul>
</nav>
</header>
<main>
@yield('main')

30
dashboard/resources/views/overview.blade.php
View File

@ -60,7 +60,7 @@ const chartExtractions = document.getElementById('extractions');
const cExtractions = echarts.init(chartExtractions);
const filters = {
regions: ["Davos", "Engadin", "Heidiland", "St. Moritz"]
regions: ["Alle", "Davos", "Engadin", "Heidiland", "St. Moritz"]
}
const cExtractionsOptions = {
@ -79,7 +79,7 @@ const cExtractionsOptions = {
xAxis: {
type: 'category',
boundaryGap: false,
data: ['2024-04-14','2024-04-15','2024-04-16','2024-04-19','2024-04-22','2024-04-25','2024-04-28','2024-05-01','2024-05-04','2024-05-07','2024-05-10','2024-05-13','2024-05-16','2024-05-19','2024-05-22','2024-05-25','2024-05-28','2024-05-31','2024-06-01','2024-06-04','2024-06-07','2024-06-10','2024-06-13','2024-06-16','2024-06-19','2024-06-22','2024-06-25','2024-06-28','2024-07-01','2024-07-04','2024-07-07','2024-07-10','2024-07-13','2024-07-16','2024-07-19','2024-07-22','2024-07-25','2024-07-28','2024-07-31','2024-08-01','2024-08-04','2024-08-07','2024-08-10','2024-08-13','2024-08-16','2024-08-19','2024-08-22']
data: {!! json_encode($growth['dates']) !!}
},
yAxis: {
type: 'value'
@ -89,35 +89,51 @@ const cExtractionsOptions = {
name: 'Alle',
type: 'line',
stack: 'Total',
data: [596, 239, 835, 673, 863, 1803, 904, 915, 958, 966, 1001, 1031, 1044, 1055, 1158, 1162, 1181, 1203, 1207, 1214, 1254, 1258, 1264, 1288, 1296, 1305, 1318, 1323, 1330, 1333, 1342, 1350, 1436, 1454, 1461, 1469, 1492, 1504, 1506, 1510, 1512, 1518, 1534, 1535, 1541, 1544, 1500]
data: {!! json_encode($growth['total_all']) !!}
},
{
name: 'Heidiland',
type: 'line',
stack: 'Heidiland',
data: [133,64,197,151,197,417,210,213,215,220,226,239,247,251,251,252,262,275,276,277,281,283,284,286,287,287,287,287,287,287,287,289,290,292,293,294,294,294,295,295,295,296,312,313,313,313,301]
data: {!! json_encode($growth['total_heidiland']) !!}
},
{
name: 'Davos',
type: 'line',
stack: 'Davos',
data: [133,56,189,152,196,409,206,209,209,209,221,223,223,224,226,227,230,234,236,238,250,252,252,259,261,263,267,270,272,272,274,274,277,278,279,281,286,289,289,289,289,290,290,290,293,296,285]
data: {!! json_encode($growth['total_davos']) !!}
},
{
name: 'Engadin',
type: 'line',
stack: 'Engadin',
data: [185,73,258,212,278,569,284,289,326,326,340,346,350,355,413,413,413,413,414,415,438,438,442,455,460,463,470,472,477,479,484,486,544,554,558,561,578,585,586,590,592,595,595,595,597,597,583]
data: {!! json_encode($growth['total_engadin']) !!}
},
{
name: 'St. Moritz',
type: 'line',
stack: 'St. Moritz',
data: [145,46,191,158,192,408,204,204,208,211,214,223,224,225,268,270,276,281,281,284,285,285,286,288,288,292,294,294,294,295,297,301,325,330,331,333,334,336,336,336,336,337,337,337,338,338,331]
data: {!! json_encode($growth['total_stmoritz']) !!}
},
]
};
cExtractions.setOption(cExtractionsOptions);
const map = L.map('leaflet').setView([46.862962, 9.535296], 9);
L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {
maxZoom: 19,
attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a>'
}).addTo(map);
const properties = {!! json_encode($geo) !!}
properties.forEach( prop => {
console.log(prop);
let coords = prop.coordinates.split(',');
L.marker(coords).addTo(map).bindPopup('<a href="/prop/'+prop.id+'">'+prop.coordinates+'</a>');
})
</script>
@endsection

19
dashboard/resources/views/property.blade.php
View File

@ -3,10 +3,23 @@
<article class="header">
<header>
<h2>
Properties pro Region
Angaben zur Property
</h2>
</header>
<div id="chart-props-per-region"></div>
<dl>
@foreach ($base as $dt => $dd)
<dt>{{ $dt }}</dt>
<dd>
@if ($dt === 'property_platform_id')
<a href="https://www.e-domizil.ch/rental/{{ $dd }}" rel="noopener noreferrer" target="_blank">
{{ $dd }}
</a>
@else
{{ $dd }}
@endif
</dd>
@endforeach
</dl>
</article>
<article class="header">
<header>
@ -24,6 +37,8 @@ const h2Belegung = document.getElementById('belegung-title');
const cCalendarOptions = {
timeline: {
data: {!! $extractiondates !!},
playInterval: 1000,
axisType: 'time',
left: 0,
right: 0,
label: {

13
dashboard/routes/web.php
View File

@ -5,6 +5,8 @@ use App\Api;
Route::get('/', function () {
$propertiesGrowth = Api::propertiesGrowth();
$propsPerRegion = Api::propertiesPerRegion();
$propsPerRegionName = [];
$propsPerRegionCounts = [];
@ -14,11 +16,15 @@ Route::get('/', function () {
$propsPerRegionCounts[] = $el['count_properties'];
}
return view('overview', ["propsPerRegion" => [json_encode($propsPerRegionName), json_encode($propsPerRegionCounts)]]);
$propertiesGeo = Api::propertiesGeo();
//dump($propertiesGeo);
return view('overview', ["geo" => $propertiesGeo, "growth" => $propertiesGrowth, "propsPerRegion" => [json_encode($propsPerRegionName), json_encode($propsPerRegionCounts)]]);
});
Route::get('/prop/{id}', function (int $id) {
$propertyBase = Api::propertyBase($id);
$extractions = Api::propertyExtractions($id);
$data = [];
$dates = [];
@ -27,8 +33,11 @@ Route::get('/prop/{id}', function (int $id) {
$series = [];
$dates[] = $ext['created_at'];
$extCalendar = json_decode($ext['calendar'], 1);
foreach ($extCalendar as $date => $status) {
$series[] = [$date, $status];
}
@ -37,5 +46,5 @@ Route::get('/prop/{id}', function (int $id) {
}
return view('property', ["extractiondates" => json_encode($dates), "calendar" => $data]);
return view('property', ['base' => $propertyBase[0], "extractiondates" => json_encode($dates), "calendar" => $data]);
});

11
etl/src/api/main.py
View File

@ -23,12 +23,19 @@ def properties_region():
@app.get("/properties/growth")
def properties_growth():
options = {"dates" : d.properties_growth().pl()['date'].to_list(), "values" : d.properties_growth().pl()['properties_count'].to_list()}
options = {"dates" : d.properties_growth().pl()['date'].to_list(), "total_all" : d.properties_growth().pl()['total_all'].to_list(), "total_heidiland" : d.properties_growth().pl()['total_heidiland'].to_list(), "total_engadin" : d.properties_growth().pl()['total_engadin'].to_list(), "total_davos" : d.properties_growth().pl()['total_davos'].to_list(), "total_stmoritz" : d.properties_growth().pl()['total_stmoritz'].to_list()}
return options
@app.get("/properties/extractions/{id}")
@app.get("/properties/geo")
def properties_geo():
return d.properties_geo().pl().to_dicts()
@app.get("/property/{id}/extractions")
def property_extractions(id: int):
return d.extractions_for(property_id = id).pl().to_dicts()
@app.get("/property/{id}/base")
def property_base_data(id: int):
return d.property_base_data(id).pl().to_dicts()

117
etl/src/data/database.py
View File

@ -46,13 +46,93 @@ class Database:
def properties_growth(self):
return self.connection.sql("""
WITH PropertiesALL AS (
SELECT
strftime(created_at, '%Y-%m-%d') AS date,
COUNT(*) as properties_count
COUNT(*) as properties_count,
SUM(properties_count) OVER (ORDER BY date) AS total
FROM
consultancy_d.properties
consultancy_d.properties p
GROUP BY
date;
date
ORDER BY
date
),
PropertiesR1 AS (
SELECT
strftime(created_at, '%Y-%m-%d') AS date,
COUNT(*) as properties_count,
SUM(properties_count) OVER (ORDER BY date) AS total
FROM
consultancy_d.properties p
WHERE
p.seed_id = 1
GROUP BY
date
ORDER BY
date
),
PropertiesR2 AS (
SELECT
strftime(created_at, '%Y-%m-%d') AS date,
COUNT(*) as properties_count,
SUM(properties_count) OVER (ORDER BY date) AS total
FROM
consultancy_d.properties p
WHERE
p.seed_id = 2
GROUP BY
date
ORDER BY
date
),
PropertiesR3 AS (
SELECT
strftime(created_at, '%Y-%m-%d') AS date,
COUNT(*) as properties_count,
SUM(properties_count) OVER (ORDER BY date) AS total
FROM
consultancy_d.properties p
WHERE
p.seed_id = 3
GROUP BY
date
ORDER BY
date
),
PropertiesR4 AS (
SELECT
strftime(created_at, '%Y-%m-%d') AS date,
COUNT(*) as properties_count,
SUM(properties_count) OVER (ORDER BY date) AS total
FROM
consultancy_d.properties p
WHERE
p.seed_id = 4
GROUP BY
date
ORDER BY
date
)
SELECT
p.date,
p.total AS total_all,
pR1.total as total_heidiland,
pR2.total AS total_davos,
pR3.total AS total_engadin,
pR4.total AS total_stmoritz
FROM
PropertiesAll p
LEFT JOIN
PropertiesR1 pR1 ON p.date = pR1.date
LEFT JOIN
PropertiesR2 pR2 ON p.date = pR2.date
LEFT JOIN
PropertiesR3 pR3 ON p.date = pR3.date
LEFT JOIN
PropertiesR4 pR4 ON p.date = pR4.date
ORDER BY
p.date
""")
def properties_per_region(self):
@ -233,9 +313,10 @@ class Database:
consultancy_d.extractions
WHERE
type == 'calendar' AND
property_id = {property_id}
property_id = {property_id} AND
calendar NOT NULL
ORDER BY
property_id
created_at
""")
# Anzahl der extrahierten properties pro Exktraktionsvorgang
@ -281,3 +362,29 @@ class Database:
ORDER BY property_id
""")
def property_base_data(self, id):
return self.connection.sql(f"""
SELECT
p.property_platform_id,
p.created_at as first_found,
p.last_found,
r.name as region_name
FROM
consultancy_d.properties p
INNER JOIN consultancy_d.seeds s ON s.id = p.seed_id
INNER JOIN consultancy_d.regions r ON s.region_id = r.id
WHERE
p.id = {id}
""")
def properties_geo(self):
return self.connection.sql("""
SELECT
p.id,
p.check_data as coordinates
FROM
consultancy_d.properties p
""")

17
etl/src/gio/test_duckdb.py
View File

@ -1,17 +0,0 @@
import data
import polars as pl
inst = data.load()
"""
test = inst.extractions_for(1).pl()
out = test.with_columns(
pl.col("calendar").str.extract_all(r"([0-9]{4}-[0-9]{2}-[0-9]{2})|[0-2]").alias("extracted_cal"),
)
out = out.drop(['calendar', 'property_id'])
print(out.to_dict(as_series=True))
"""
print(inst.price_developement_per_property().pl())