typo

Kommentare gelöscht und haversineFormel angepasst, dass sie gleich wie in der Quelle ist

dashboard/app/Api.php

@@ -5,6 +5,11 @@ namespace App;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Http;
 
+/*
+ * Class contains methods which make calls to the API.
+ * Successfull calls get cached.
+ */
+
 class Api
 {
 
@@ -14,12 +19,15 @@ class Api
         $endpoint = env('FASTAPI_URI');
         $request = $endpoint.$path;
 
+        // load from cache if available
         if (Cache::has($request)) {
-            // return Cache::get($request);
+            return Cache::get($request);
         }
 
-        $get = Http::timeout(1600)->get($request);
+        // Set timeout to .5h
+        $get = Http::timeout(1800)->get($request);
 
+        // return result and cache it
         if($get->successful()){
             $result = $get->json();
             Cache::put($request, $result);
@@ -29,81 +37,79 @@ class Api
         return null;
     }
 
-    public static function propertiesPerRegion()
-    {
-        return self::get('/region/properties');
-    }
-
-    public static function propertiesGrowth()
+    public static function propertiesGrowth(): mixed
     {
         return self::get('/properties/growth');
     }
 
-     public static function propertiesGeo()
+     public static function propertiesGeo(): mixed
     {
         return self::get('/properties/geo');
     }
 
-    public static function propertyExtractions(int $id)
+    public static function propertyExtractions(int $id): mixed
     {
-        return self::get("/property/{$id}/extractions");
+        return self::get("/properties/{$id}/extractions");
     }
 
-    public static function propertyCapacities(int $id)
+    public static function propertyCapacities(int $id): mixed
     {
-        return self::get("/property/{$id}/capacities");
+        return self::get("/properties/{$id}/capacities");
     }
 
     public static function propertyBase(int $id): mixed
     {
-        return self::get("/property/{$id}/base");
-    }
-
-    public static function regionBase(int $id): mixed
-    {
-        return self::get("/region/{$id}/base");
-    }
-
-    public static function regionPropertiesCapacities(int $id): mixed
-    {
-        return self::get("/region/{$id}/properties/capacities");
-    }
-
-    public static function regionCapacitiesMonthly(int $id, string $date): mixed
-    {
-        return self::get("/region/{$id}/capacities/monthly/{$date}");
+        return self::get("/properties/{$id}/base");
     }
 
     public static function propertyCapacitiesMonthly(int $id, string $date): mixed
     {
-        return self::get("/property/{$id}/capacities/monthly/{$date}");
-    }
-
-    public static function regionCapacitiesDaily(int $id, string $date): mixed
-    {
-        return self::get("/region/{$id}/capacities/weekdays/{$date}");
+        return self::get("/properties/{$id}/capacities/monthly/{$date}");
     }
 
     public static function propertyCapacitiesDaily(int $id, string $date): mixed
     {
-        return self::get("/property/{$id}/capacities/weekdays/{$date}");
+        return self::get("/properties/{$id}/capacities/daily/{$date}");
     }
 
     public static function propertyNeighbours(int $id): mixed
     {
-        return self::get("/property/{$id}/neighbours");
+        return self::get("/properties/{$id}/neighbours");
+    }
+
+    public static function regions(): mixed
+    {
+        return self::get('/regions');
+    }
+
+    public static function regionBase(int $id): mixed
+    {
+        return self::get("/regions/{$id}/base");
+    }
+
+    public static function regionPropertiesCapacities(int $id): mixed
+    {
+        return self::get("/regions/{$id}/properties/capacities");
+    }
+
+    public static function regionCapacitiesMonthly(int $id, string $date): mixed
+    {
+        return self::get("/regions/{$id}/capacities/monthly/{$date}");
+    }
+
+    public static function regionCapacitiesDaily(int $id, string $date): mixed
+    {
+        return self::get("/regions/{$id}/capacities/daily/{$date}");
     }
 
     public static function regionCapacities(int $id): mixed
     {
-        return self::get("/region/{$id}/capacities");
+        return self::get("/regions/{$id}/capacities");
     }
 
     public static function regionMovingAverage(int $id, string $date): mixed
     {
-        return self::get("/region/{$id}/movingAverage/{$date}");
+        return self::get("/regions/{$id}/moving-average/{$date}");
     }
     
-
-    
 }

dashboard/resources/css/app.css

@@ -108,6 +108,20 @@ button[popovertarget]>span{
   font-size: 1em;
 }
 
+[popover] h3{
+  font-size: .95em;
+  margin-top: 1em;
+}
+
+p.formula{
+  font-family: monospace;
+  border: 1px solid #aaa;
+  padding: .5em 1em;
+}
+
+p + p{
+  margin-top: 1em;
+}
 
 /*
   9. Create a root stacking context
@@ -135,21 +149,32 @@ body>header>nav{
   background: #fff;
   border-radius: .2em;
   position: relative;
+  border: 1px solid #fff;
 }
 
 body>header>nav>ul{
   position: absolute;
   background: #fff;
-  width: 100%;
+  width: calc(100% + 2px);
   list-style: none;
-  padding: 0;
+  padding: 0 0 1em;
   top: -999em;
   left: -999em;
+  border-radius: 0 0 .2em .2em;
+  border-left: 1px solid #aaa;
+  border-right: 1px solid #aaa;
+  border-bottom: 1px solid #aaa;
+}
+
+body>header>nav:hover{
+  border-radius: .2em .2em 0 0;
+  border: 1px solid #aaa;
+
 }
 
 body>header>nav:hover ul{
   top: initial;
-  left: 0;
+  left: -1px;
 }
 
 body>header>nav>ul>li a,
@@ -180,10 +205,10 @@ body.overview main{
   grid-template-columns: repeat(8, minmax(1%, 50%));
   grid-template-rows: repeat(4, 1fr);
   grid-template-areas:
-      "chart3 chart3 chart3 chart1 chart1 chart1 chart4 chart4"
-      "chart3 chart3 chart3 chart1 chart1 chart1 chart4 chart4"
-      "chart3 chart3 chart3 chart2 chart2 chart2 chart4 chart4"
-      "chart3 chart3 chart3 chart2 chart2 chart2 chart4 chart4"
+      "chart1 chart1 chart1 chart2 chart2 chart2 chart4 chart4"
+      "chart1 chart1 chart1 chart2 chart2 chart2 chart4 chart4"
+      "chart1 chart1 chart1 chart3 chart3 chart3 chart4 chart4"
+      "chart1 chart1 chart1 chart3 chart3 chart3 chart4 chart4"
 }
 
 body.region main{
@@ -203,8 +228,8 @@ body.property main{
   grid-template-columns: repeat(4, minmax(10%, 50%));
     grid-template-rows: repeat(4, 1fr) 4em;
     grid-template-areas:
-      "chart2 chart2 chart1 chart1"
-      "chart2 chart2 chart1 chart1"
+      "chart1 chart1 chart2 chart2"
+      "chart1 chart1 chart2 chart2"
       "chart5 chart5 chart3 chart4"
       "chart5 chart5 chart3 chart4"
       "chart5 chart5 timeline timeline";
@@ -249,10 +274,28 @@ article>header>h2{
     height: auto;
   }
 
-  main{
+  body.overview main,
+  body.region main,
+  body.property main{
     height: auto;
     grid-template-columns: 100%;
-    grid-template-rows: repeat(4, minmax(20em, 25em));
+    grid-template-rows: repeat(5, minmax(20em, 25em)) 4em;
+    grid-template-areas: "chart1" "chart2" "chart3" "chart4" "chart5" "chart6" "timeline";
+  }
+
+  body.overview main{
+    grid-template-rows: minmax(20em, 40em) repeat(3, minmax(20em, 25em));
+    grid-template-areas: "chart1" "chart2" "chart3" "chart4";
+  }
+
+  body.region main{
+    grid-template-rows: minmax(20em, 40em) repeat(4, minmax(20em, 25em)) 4em;
+    grid-template-areas: "chart1" "chart2" "chart3" "chart4" "chart6" "timeline";
+  }
+
+  body.property main{
+    grid-template-rows: repeat(5, minmax(20em, 25em)) 4em;
+    grid-template-areas: "chart1" "chart2" "chart3" "chart4" "chart5" "timeline";
   }
 }
 

dashboard/resources/views/overview.blade.php

@@ -5,34 +5,44 @@
   <strong>Start</strong>
   <ul>
     @foreach($regions as $r)
-    <li><a href="/region/{{ $r['region_id'] }}">{{ $r['region_name'] }}</a></li>
+    <li><a href="/region/{{ $r['id'] }}">{{ $r['name'] }}</a></li>
     @endforeach
   </ul>
 </nav>
 @endsection
 @section('main')
-<article class="header" style="grid-area: chart3;">
+<article class="header" style="grid-area: chart1;">
   <header>
-    <h2>Auslastung aller Mietobjekte über Gesamte Zeit</h2>
+    <h2>Verfügbarkeit aller Mietobjekte über gesamten beobachteten Zeitraum</h2>
     <button popovertarget="pop1">
       <span>Erklärungen zum Diagramm</span>
     </button>
     <div popover id="pop1">
-      <h2>Auslastung aller Mietobjekte über Gesamte Zeit</h2>
+      <h2>Verfügbarkeit aller Mietobjekte über gesamten beobachteten Zeitraum</h2>
       <p>
-        Das Diagramm gibt eine Übersicht, wie die Auslastung von Mietobjekten am Datum des Scrapings waren. Dazu wird für jedes Mietobjekt die durchschnittliche Verfügbarkeit ermittelt.
+        Das Diagramm zeigt die Verfügbarkeit aller Mietobjekte zu allen beobachteten Zeitpunkten.
       </p>
       <ul>
-        <li>X-Achse: Zeitpunkt Scraping.</li>
+        <li>X-Achse: Zeitpunkt Beobachtung.</li>
         <li>Y-Achse: Mietobjekte.</li>
-        <li>Kategorien: 0% = Das Mietobjekt ist komplett verfügbar; 100% = Das Mietobjekt ist komplett ausgebucht.</li>
+        <li>Kategorien: 0% = Das Mietobjekt ist komplett Ausgebucht; 100% = Das Mietobjekt kann zu allen Verfügbaren Daten gebucht werden.</li>
+      </ul>
+
+      <h3>Berrechnung Verfügbarkeit</h3>
+      <p>Die Verfügbarkeit eines Mietobjekt errechnet sich folgendermassen:</p>
+      <p class="formula">
+        Verfügbarkeit = (100 / (Anzahl Buchungsdaten * 2)) * Summe Status
+      </p>
+      <ul>
+        <li>Status: Jeder verfügbare Kalendertag kann den Status «Nicht Verfügbar» (0), «Verfügbar (kein Anreisetag)» (1) oder «Verfügbar» (2) aufweisen.</li>
+        <li>Anzahl Buchungsdaten: Die Summe aller angebotenen Buchungsdaten mit zwei multipliziert (= Alle Buchungdaten haben den Status «Verfügbar»)</li>
       </ul>
     </div>
     <div>
   </header>
   <div id="chart-heatmap"></div>
 </article>
-<article class="header" style="grid-area: chart1;">
+<article class="header" style="grid-area: chart2;">
   <header>
     <h2>
       Anzahl jemals gefundene Kurzzeitmietobjekte pro Region
@@ -41,20 +51,20 @@
       <span>Erklärungen zum Diagramm</span>
     </button>
     <div popover id="pop2">
-      <h2>Anzahl jemals gefundene Kurzzeitmietobjekte pro Region</h2>
+      <h2>Anzahl jemals gefundener Mietobjekte pro Region</h2>
       <p>
-        Das Balkendiagramm zeigt wieviele Kurzzeitmietobjekte insgesamt pro Region über den gesamten Datenerhebungszeitraum, gefunden wurden.
+        Das Balkendiagramm zeigt die Anzahl jemals gefundener Mietobjekte pro Region.
       </p>
       <ul>
-        <li>X-Achse: Bezeichnung der Region.</li>
-        <li>Y-Achse: Anzahl Mietobjekte.</li>
+        <li>X-Achse: Region</li>
+        <li>Y-Achse: Anzahl Mietobjekte</li>
       </ul>
     </div>
     <div>
   </header>
   <div id="chart-props-per-region"></div>
 </article>
-<article class="header" style="grid-area: chart2;">
+<article class="header" style="grid-area: chart3;">
   <header>
     <h2>
       Entwicklung der Anzahl jemals gefunden Kurzzeitmietobjekte
@@ -63,13 +73,13 @@
       <span>Erklärungen zum Diagramm</span>
     </button>
     <div popover id="pop3">
-      <h2>Entwicklung Anzahl jemals gefundener Kurzzeitmietobjekte pro Region</h2>
+      <h2>Entwicklung Anzahl jemals gefundener Mietobjekte pro Region</h2>
       <p>
-        Das Liniendiagramm zeigt die Entwicklung der gefundenen Mietobjekte pro Region.
+        Das Liniendiagramm zeigt die Entwicklung aller jemals gefundener Mietobjekte pro Region.
       </p>
       <ul>
-        <li>X-Achse: Zeitpunkt Scraping.</li>
-        <li>Y-Achse: Anzahl Mietobjekte.</li>
+        <li>X-Achse: Zeitpunkt Beobachtung</li>
+        <li>Y-Achse: Anzahl Mietobjekte</li>
       </ul>
     </div>
     <div>
@@ -82,7 +92,7 @@
 <script type="module">
 const sharedOptions = {
   basic: {
-    color: {!! $chartOptions['colors'] !!},
+    color: {!! $diagramsOptions['shared']['colors'] !!},
     grid: {
       top: 30,
       left: 70,
@@ -101,7 +111,7 @@ const sharedOptions = {
     }
   }
 }
-const extractionDates = {!! json_encode($regionPropertiesCapacities['scrapeDates']) !!};
+const extractionDates = {!! $diagramsOptions['shared']['extractionDates'] !!};
 
 const chartHeatmap = document.getElementById('chart-heatmap');
 const cHeatmap = echarts.init(chartHeatmap);
@@ -111,10 +121,13 @@ const cHeatmapOptions = {
     position: 'top'
   },
   grid: {
+    show: true,
+    borderWidth: 1,
+    borderColor: '#aaa',
     top: 30,
     right: 45,
-    bottom: 50,
-    left: 5
+    bottom: 70,
+    left: 30
   },
   dataZoom: [{
       type: 'slider'
@@ -125,23 +138,52 @@ const cHeatmapOptions = {
       yAxisIndex: 0,
   }],
   xAxis: {
-    show: false,
-    name: 'Kurzzeitmietobjekt',
+    show: true,
+    name: 'Zeitpunkt Beobachtung',
     type: 'category',
     data: extractionDates,
     splitArea: {
       show: false
     },
+    splitArea: {
+      show: false
+    },
     axisLabel: {
-      show: true,
+      show: false,
+    },
+    axisTick: {
+        show: false,
+    },
+    axisLine: {
+        show: false,
+    },
+    nameLocation: 'center',
+    nameGap: 10,
+    nameTextStyle: {
+      fontWeight: 'bold',
     }
   },
   yAxis: {
-    show: false,
+    show: true,
     type: 'category',
-    data: {!! json_encode($regionPropertiesCapacities['property_ids']) !!},
+    data: {!! $diagramsOptions['heatmap']['yAxis']['data'] !!},
     splitArea: {
-      show: true
+      show: false
+    },
+    axisTick: {
+        show: false,
+    },
+    axisLine: {
+        show: false,
+    },
+    axisLabel: {
+      show: false,
+    },
+    name: 'Mietobjekte',
+    nameLocation: 'center',
+    nameGap: 10,
+    nameTextStyle: {
+      fontWeight: 'bold',
     }
   },
   visualMap: {
@@ -161,16 +203,16 @@ const cHeatmapOptions = {
   },
   series: [
     {
-      name: 'Auslastung',
+      name: 'Verfügbarkeit',
       type: 'heatmap',
       blurSize: 0,
-      data: {!! json_encode($regionPropertiesCapacities['values']) !!},
+      data: {!! $diagramsOptions['heatmap']['series']['data'] !!},
       label: {
         show: false
       },
       tooltip: {
         formatter: (data) => {
-          return `Kurzzeitmietobjekte-ID: ${data.data[1]}<br />Datum Scraping: ${data.data[0]}<br/>Auslastung: ${data.data[2].toFixed(2)} %`
+          return `Kurzzeitmietobjekte-ID: ${data.data[1]}<br />Beobachtungszeitpunkt: ${data.data[0]}<br/>Verfügbarkeit: ${data.data[2].toFixed(2)} %`
         },
       },
       emphasis: {
@@ -198,7 +240,7 @@ const cPropsPerRegionOptions = {
       fontWeight: 'bold',
     },
     type: 'category',
-    data: {!! $propsPerRegion[1] !!}
+    data: {!! $diagramsOptions['propertiesPerRegion']['xAxis']['data'] !!}
   },
   yAxis: {
     type: 'value',
@@ -211,7 +253,7 @@ const cPropsPerRegionOptions = {
   },
   series: [
     {
-      data: {!! $propsPerRegion[2] !!},
+      data: {!! $diagramsOptions['propertiesPerRegion']['yAxis']['data'] !!},
       type: 'bar',
       itemStyle: {
         color: (e) => {
@@ -227,21 +269,17 @@ cPropsPerRegion.setOption(cPropsPerRegionOptions);
 const chartExtractions = document.getElementById('extractions');
 const cExtractions = echarts.init(chartExtractions);
 
-const filters = {
-  regions: ["Alle", "Davos", "Engadin", "Heidiland", "St. Moritz"]
-}
-
 const cExtractionsOptions = {
   color: sharedOptions.basic.color,
   tooltip: {
     trigger: 'axis'
   },
   legend: {
-    data: filters.regions
+    show: true
   },
   grid: sharedOptions.basic.grid,
   xAxis: {
-    name: 'Zeitpunkt Scraping',
+    name: 'Zeitpunkt Beobachtung',
     nameLocation: 'center',
     nameGap: 24,
     nameTextStyle: {
@@ -265,27 +303,31 @@ const cExtractionsOptions = {
       name: 'Alle',
       type: 'line',
       stack: 'Total',
-      data: {!! json_encode($growth['total_all']) !!},
+      data: {!! json_encode($diagramsOptions['extractions']['series']['total_all']) !!},
     },
     {
+      connectNulls: true,
       name: 'Davos',
       type: 'line',
-      data: {!! json_encode($growth['total_davos']) !!}
+      data: {!! json_encode($diagramsOptions['extractions']['series']['total_davos']) !!}
     },
     {
+      connectNulls: true,
       name: 'Engadin',
       type: 'line',
-      data: {!! json_encode($growth['total_engadin']) !!}
+      data: {!! json_encode($diagramsOptions['extractions']['series']['total_engadin']) !!}
     },
     {
+      connectNulls: true,
       name: 'Heidiland',
       type: 'line',
-      data: {!! json_encode($growth['total_heidiland']) !!}
+      data: {!! json_encode($diagramsOptions['extractions']['series']['total_heidiland']) !!}
     },
     {
+      connectNulls: true,
       name: 'St. Moritz',
       type: 'line',
-      data: {!! json_encode($growth['total_stmoritz']) !!}
+      data: {!! json_encode($diagramsOptions['extractions']['series']['total_stmoritz']) !!}
     },
   ]
 };
@@ -318,12 +360,6 @@ cHeatmap.on('click', 'series', (e) => {
   window.open(`/property/${e.value[1]}?date=${e.value[0]}`, '_self');
 })
 
-cPropsPerRegion.on('click', 'series', (e) => {
-  console.log(e.dataIndex);
-  //window.open(`/property/${e.value[1]}?date=${e.value[0]}`, '_self');
-})
-
-
 
 </script>
 @endsection

dashboard/resources/views/property-nodata.blade.php

@@ -0,0 +1,27 @@
+@extends('base')
+@section('body-class', 'property')
+@section('header')
+<nav>
+  <strong>Property: {{ $base['check_data'] }} ({{ $base['region_name'] }})</strong>
+  <ul>
+    <li><a href="/">Start</a></li>
+    @foreach($regions as $r)
+    <li><a href="/region/{{ $r['id'] }}">{{ $r['name'] }}</a></li>
+    @endforeach
+  </ul>
+</nav>
+<button popovertarget="prop-details"></button>
+<div popover id="prop-details">
+    <dl>
+        <dt>Region</dt>
+        <dd>{{ $base['region_name'] }}</dd>
+        <dt>Zum ersten mal gefunden</dt>
+        <dd>{{ $base['first_found'] }}</dd>
+        <dt>Zum letzten mal gefunden</dt>
+        <dd>{{ $base['last_found'] }}</dd>
+    </dl>
+</div>
+@endsection
+@section('main')
+<p>Für dieses Mietobjekt sind keine Daten vorhanden.</p>
+@endsection

dashboard/resources/views/property.blade.php

@@ -2,11 +2,11 @@
 @section('body-class', 'property')
 @section('header')
 <nav>
-  <strong>Property: {{ $base['check_data'] }}</strong>
+  <strong>Mietobjekt: {{ $base['latlng'] }} ({{ $base['region_name'] }})</strong>
   <ul>
     <li><a href="/">Start</a></li>
     @foreach($regions as $r)
-    <li><a href="/region/{{ $r['region_id'] }}">{{ $r['region_name'] }}</a></li>
+    <li><a href="/region/{{ $r['id'] }}">{{ $r['name'] }}</a></li>
     @endforeach
   </ul>
 </nav>
@@ -27,10 +27,10 @@
 <article style="grid-area: timeline;">
     <div id="timeline"></div>
 </article>
-<article class="header" style="grid-area: chart1;">
+<article class="header" style="grid-area: chart2;">
     <header>
         <h2 id="belegung-title">
-            Kalenderansicht der Belegung am <span class="date">{{ $startDate }}</span>
+            Kalenderansicht der Verfügbarkeit am <span class="date">{{ $startDate }}</span>
         </h2><button popovertarget="popup-cal"></button>
         <div popover id="popup-cal">
             <p>
@@ -51,13 +51,13 @@
 <article class="header" style="grid-area: chart3;">
     <header>
         <h2>
-            Belegung Mietobjekt Monate am <span class="date">{{ $startDate }}</span>
+            Verfügbarkeit Mietobjekt Monate am <span class="date">{{ $startDate }}</span>
         </h2>
     </header>
     <div id="chart-capacity-monthly">
     </div>
 </article>
-<article class="header" style="grid-area: chart2;">
+<article class="header" style="grid-area: chart1;">
     <header>
         <h2>
             Entwicklung der Verfügbarkeit
@@ -73,7 +73,7 @@
 <article class="header" style="grid-area: chart4;">
     <header>
         <h2>
-            Belegung Mietobjekt Tage am <span class="date">{{ $startDate }}</span>
+            Verfügbarkeit Mietobjekt Tage am <span class="date">{{ $startDate }}</span>
         </h2>
     </header>
     <div id="chart-capacity-daily">
@@ -81,41 +81,41 @@
 <script type="module">
 
 const sharedOptions = {
-  basic: {
-    color: {!! $chartOptions['colors'] !!},
-    grid: {
-      top: 20,
-      left: 60,
-      right: 0,
-      bottom: 50
-    },
-    tooltip: {
-        show: true,
-        trigger: 'axis',
-        valueFormatter: (value) => value.toFixed(2)+' %'
-    },
-    name: (opt) => {
-      return {
-        name: opt.name,
-        nameLocation: opt.location,
-        nameGap: 24,
-        nameTextStyle: {
-          fontWeight: 'bold',
+        extractiondates: {!! $diagramsOptions['shared']['extractiondates']!!},
+        basic: {
+            color: {!!$diagramsOptions['shared']['colors']!!},
+        grid: {
+            top: 20,
+            left: 60,
+            right: 0,
+            bottom: 50
         },
-      }
+        tooltip: {
+            show: true,
+            trigger: 'axis',
+            valueFormatter: (value) => value.toFixed(2) + ' %'
+        },
+        name: (opt) => {
+            return {
+                name: opt.name,
+                nameLocation: opt.location,
+                nameGap: 24,
+                nameTextStyle: {
+                    fontWeight: 'bold',
+                }
+            }
+        }
     }
-  }
 }
 
 const chartTimeline = document.getElementById('timeline');
 const cTimeline = echarts.init(chartTimeline);
-
 const cTimelineOptions = {
     grid: {
         show: false,
     },
     timeline: {
-        data: {!! $extractiondates !!},
+        data: sharedOptions.extractiondates,
         playInterval: 1000,
         axisType: 'time',
         left: 8,
@@ -136,7 +136,7 @@ const cCapacityMonthlyOptions = {
     tooltip: sharedOptions.basic.tooltip,
     timeline: {
         show: false,
-        data: {!! $extractiondates !!},
+        data: sharedOptions.extractiondates,
         axisType: 'time',
     },
     grid: {
@@ -148,7 +148,7 @@ const cCapacityMonthlyOptions = {
     xAxis: {
         type: 'value',
         max: 100,
-        name: 'Auslastung in %',
+        name: 'Verfügbarkeit in %',
         nameLocation: 'center',
         nameGap: 25,
             nameTextStyle: {
@@ -159,7 +159,7 @@ const cCapacityMonthlyOptions = {
         type: 'category',
     },
     options: [
-        @foreach ($capacitiesMonthly as $cM)
+        @foreach ($diagramsOptions['capacityMonthly']['options'] as $cM)
         {
         yAxis: {
             data: {!! json_encode($cM['months']) !!}
@@ -185,7 +185,7 @@ const cCapacityDailyOptions = {
     tooltip: sharedOptions.basic.tooltip,
     timeline: {
         show: false,
-        data: {!! $extractiondates !!},
+        data: sharedOptions.extractiondates,
         axisType: 'time',
     },
     grid: {
@@ -197,7 +197,7 @@ const cCapacityDailyOptions = {
     xAxis: {
         type: 'value',
         max: 100,
-        name: 'Auslastung in %',
+        name: 'Verfügbarkeit in %',
         nameLocation: 'center',
         nameGap: 25,
             nameTextStyle: {
@@ -208,7 +208,7 @@ const cCapacityDailyOptions = {
         type: 'category',
     },
     options: [
-        @foreach ($capacitiesDaily as $cD)
+        @foreach ($diagramsOptions['capacityDaily']['options'] as $cD)
         {
         yAxis: {
             data: {!! json_encode($cD['weekdays']) !!}
@@ -233,7 +233,7 @@ const cCapacity = echarts.init(chartCapacity);
 const cCapacityOptions = {
     color: sharedOptions.basic.color,
     legend: {
-        data: ['Auslastung Property', 'Auslastung {{ $base['region_name'] }}', 'Auslastung alle Regionen']
+        show: true
     },
   tooltip: {
     trigger: 'axis',
@@ -249,8 +249,8 @@ const cCapacityOptions = {
   xAxis: {
     type: 'category',
     boundaryGap: false,
-    data: {!! json_encode($propertyCapacities['dates']) !!},
-    name: 'Zeitpunkt Scraping',
+    data: {!! $diagramsOptions['capacities']['xAxis']['data'] !!},
+    name: 'Zeitpunkt Beobachtung',
     nameLocation: 'center',
     nameGap: 24,
     nameTextStyle: {
@@ -261,7 +261,7 @@ const cCapacityOptions = {
     type: 'value',
     min: 0,
     max: 100,
-    name: 'Auslastung in Prozent',
+    name: 'Verfügbarkeit in %',
     nameLocation: 'center',
     nameGap: 38,
     nameTextStyle: {
@@ -270,22 +270,22 @@ const cCapacityOptions = {
   },
   series: [
     {
-          name: 'Auslastung Property',
+          name: 'Verfügbarkeit Mietobjekt',
           type: 'line',
           symbolSize: 7,
-          data: {!! json_encode($propertyCapacities['capacities']) !!}
+          data: {!! $diagramsOptions['capacities']["series"][0]["data"] !!}
         },
         {
-            name: 'Auslastung {{ $base['region_name'] }}',
+            name: 'Verfügbarkeit {{ $base['region_name'] }}',
             type: 'line',
             symbolSize: 7,
-            data: {!! json_encode($regionCapacities[0]) !!}
+            data: {!! $diagramsOptions['capacities']["series"][1]["data"] !!}
         },
         {
-            name: 'Auslastung alle Regionen',
+            name: 'Verfügbarkeit alle Regionen',
             type: 'line',
             symbolSize: 7,
-            data: {!! json_encode($regionCapacities[1]) !!}
+            data: {!! $diagramsOptions['capacities']["series"][2]["data"] !!}
         }
     ]
 };
@@ -299,7 +299,7 @@ const h2Belegung = document.getElementById('belegung-title');
 const cCalendarOptions = {
     timeline: {
         show: false,
-        data: {!! json_encode($propertyCapacities['dates']) !!},
+        data: sharedOptions.extractiondates,
         axisType: 'time',
     },
     visualMap: {
@@ -330,6 +330,9 @@ const cCalendarOptions = {
           right: 10,
           bottom: '65%',
           left: 50,
+          dayLabel: {
+            fontSize: 10
+          }
         },
         {
           orient: 'horizontal',
@@ -338,6 +341,9 @@ const cCalendarOptions = {
           right: 10,
           bottom: '33%',
           left: 50,
+          dayLabel: {
+            fontSize: 10
+          }
         },
         {
           orient: 'horizontal',
@@ -346,10 +352,13 @@ const cCalendarOptions = {
           right: 10,
           bottom: '1%',
           left: 50,
+          dayLabel: {
+            fontSize: 10
+          }
         }
     ],
     options: [
-        @foreach ($calendar as $c)
+        @foreach ($diagramsOptions['calendar']['series'] as $c)
         {
             series: [{
                 type: 'heatmap',
@@ -431,7 +440,7 @@ function icon(id = 0){
 }
 
 const markers = L.featureGroup([
-    L.marker([{{ $base['check_data'] }}], {icon: icon(1)}),
+    L.marker([{{ $base['latlng'] }}], {icon: icon(1)}),
     @foreach($neighbours as $n)
     L.marker([{{ $n['lat'] }}, {{ $n['lon'] }}], {icon: icon()}).bindPopup('<a href="/property/{{ $n['id'] }}">{{ $n['lat'] }}, {{ $n['lon'] }}</a>'),
     @endforeach

dashboard/resources/views/region.blade.php

@@ -2,12 +2,12 @@
 @section('body-class', 'region')
 @section('header')
 <nav>
-  <strong>{{ $region[0]['region_name'] }}</strong>
+  <strong>{{ $region['name'] }}</strong>
   <ul>
     <li><a href="/">Start</a></li>
     @foreach($regions as $r)
-    @if($r['region_id'] != $region_id)
-    <li><a href="/region/{{ $r['region_id'] }}">{{ $r['region_name'] }}</a></li>
+    @if($r['id'] != $regionId)
+    <li><a href="/region/{{ $r['id'] }}">{{ $r['name'] }}</a></li>
     @endif
     @endforeach
   </ul>
@@ -19,20 +19,51 @@
 </article>
 <article class="header" style="grid-area: chart6;">
     <header>
-        <h2 id="prediction-title">Gleitender Mittelwert für die Auslastung der Region</h2>
+        <h2 id="prediction-title">Gleitender Mittelwert für die Verfügbarkeit der Region</h2>
+        <button popovertarget="chart-prediction-popover"></button>
+        <div id="chart-prediction-popover" popover>
+            <h2>Gleitender Mittelwert für die Verfügbarkeit der Region</h2>
+            <p>Das Diagramm...</p>
+            <ul>
+                <li>X-Achse: Zeitpunkt der Beobachtung</li>
+                <li>Y-Achse: Verfügbarkeit einer Region. 0% = Alle Mietobjekte der Region sind komplett ausgebucht; 100% = Alle Mietobjekte der Region können zu allen verfügbaren Daten gebucht werden. </li>
+            </ul>
+        </div>
     </header>
     <div id="chart-prediction"></div>
 </article>
 <article class="header" style="grid-area: chart1;">
     <header>
-        <h2 id="belegung-title">Auslastung aller Mietobjekte über Gesamte Zeit der Region</h2>
+        <h2 id="belegung-title">Verfügbarkeit aller Mietobjekte der Region über gesamten beobachteten Zeitraum</h2>
+        <button popovertarget="popup-heat"></button><div popover id="popup-heat">
+      <h2>Verfügbarkeit aller Mietobjekte der Region über gesamten beobachteten Zeitraum</h2>
+      <p>
+        Das Diagramm zeigt die Verfügbarkeit aller Mietobjekte der Region zu allen beobachteten Zeitpunkten.
+      </p>
+      <ul>
+        <li>X-Achse: Zeitpunkt Beobachtung.</li>
+        <li>Y-Achse: Mietobjekte.</li>
+        <li>Kategorien: 0% = Das Mietobjekt ist komplett Ausgebucht; 100% = Das Mietobjekt kann zu allen Verfügbaren Daten gebucht werden.</li>
+      </ul>
+
+      <h3>Berrechnung Verfügbarkeit</h3>
+      <p>Die Verfügbarkeit eines Mietobjekt errechnet sich folgendermassen:</p>
+      <p class="formula">
+        Verfügbarkeit = (100 / (Anzahl Buchungsdaten * 2)) * Summe Status
+      </p>
+      <ul>
+        <li>Status: Jeder verfügbare Kalendertag kann den Status «Nicht Verfügbar» (0), «Verfügbar (kein Anreisetag)» (1) oder «Verfügbar» (2) aufweisen.</li>
+        <li>Anzahl Buchungsdaten: Die Summe aller angebotenen Buchungsdaten mit zwei multipliziert (= Alle Buchungdaten haben den Status «Verfügbar»)</li>
+      </ul>
+    </div>
+    <div>
     </header>
     <div id="chart-heatmap"></div>
 </article>
 <article class="header" style="grid-area: chart3;">
     <header>
         <h2>
-            Auslastung Region nach Monat am <span class="date">{{ $startDate }}</span>
+            Verfügbarkeit nach Monat am <span class="date">{{ $startDate }}</span>
         </h2>
     </header>
     <div id="chart-capacity-monthly">
@@ -41,12 +72,16 @@
 <article class="header" style="grid-area: chart2;">
     <header>
         <h2>
-            Entwicklung der Auslastung
+            Entwicklung der Verfügbarkeit
         </h2>
         <button popovertarget="chart-capacity-popover"></button>
         <div id="chart-capacity-popover" popover>
-            <h2>Erkläung zum Diagramm «Entwicklung der Auslastung»</h2>
-            <p>Das Liniendiagramm zeigt die Auslastung von Regionen. 100 % = die Region ist kaum ausgelastet; 100 % der Mietobjekte sind verfügbar. 0 % = Die Region ist komplett ausgelastet; Es stehen keine Mietangebote zur Verfügung.</p>
+            <h2>Entwicklung der Verfügbarkeit</h2>
+            <p>Das Liniendiagramm zeigt die Entwicklung Verfügbarkeit der Region im Vergleich zu allen Regionen an.</p>
+            <ul>
+                <li>X-Achse: Zeitpunkt der Beobachtung</li>
+                <li>Y-Achse: Verfügbarkeit einer Region. 0% = Alle Mietobjekte der Region sind komplett ausgebucht; 100% = Alle Mietobjekte der Region können zu allen verfügbaren Daten gebucht werden. </li>
+            </ul>
         </div>
     </header>
     <div id="chart-capacity"></div>
@@ -54,7 +89,7 @@
 <article class="header" style="grid-area: chart4;">
     <header>
         <h2>
-            Auslastung Wochentage am <span class="date">{{ $startDate }}</span>
+            Verfügbarkeit nach Wochentage am <span class="date">{{ $startDate }}</span>
         </h2>
     </header>
     <div id="chart-capacity-daily">
@@ -64,7 +99,7 @@
 
 const sharedOptions = {
   basic: {
-    color: {!! $chartOptions['colors'] !!},
+    color: {!! $diagramsOptions['shared']['colors'] !!},
     grid: {
       top: 20,
       left: 60,
@@ -74,7 +109,7 @@ const sharedOptions = {
     tooltip: {
         show: true,
         trigger: 'axis',
-        valueFormatter: (value) => value.toFixed(2)+' %'
+        valueFormatter: (value) => value == null ? 'N/A' : value.toFixed(2)+' %'
     },
     name: (opt) => {
       return {
@@ -108,8 +143,8 @@ const cCapacityOptions = {
   xAxis: {
     type: 'category',
     boundaryGap: false,
-    data: {!! json_encode($regionCapacities['region']['dates']) !!},
-    name: 'Zeitpunkt Scraping',
+    data: {!! $diagramsOptions['capacity']['xAxis']['data'] !!},
+    name: 'Zeitpunkt Beobachtung',
     nameLocation: 'center',
     nameGap: 24,
     nameTextStyle: {
@@ -120,7 +155,7 @@ const cCapacityOptions = {
     type: 'value',
     min: 0,
     max: 100,
-    name: 'Auslastung in %',
+    name: 'Verfügbarkeit in %',
     nameLocation: 'center',
     nameGap: 38,
     nameTextStyle: {
@@ -128,16 +163,16 @@ const cCapacityOptions = {
     }
   },
   series: [{
-      name: 'Auslastung alle Regionen',
+      name: 'Verfügbarkeit alle Regionen',
       type: 'line',
       symbolSize: 7,
-      data: {!! json_encode($regionCapacities['all']['capacities']) !!}
+      data: {!! $diagramsOptions['capacity']['series']['all']['data'] !!}
     },
     {
-        name: 'Auslastung Region',
+        name: 'Verfügbarkeit Region',
         type: 'line',
         symbolSize: 7,
-        data: {!! json_encode($regionCapacities['region']['capacities']) !!}
+        data: {!! $diagramsOptions['capacity']['series']['region']['data'] !!}
     }]
 };
 
@@ -149,7 +184,7 @@ const cCapacityMonthly = echarts.init(chartCapacityMonthly);
 const cCapacityMonthlyOptions = {
     timeline: {
         show: false,
-        data: {!! json_encode($regionCapacities['region']['dates']) !!},
+        data: {!! $diagramsOptions['capacity']['xAxis']['data'] !!},
         axisType: 'time',
     },
     grid: {
@@ -161,7 +196,7 @@ const cCapacityMonthlyOptions = {
     xAxis: {
         type: 'value',
         max: 100,
-        name: 'Auslastung in %',
+        name: 'Verfügbarkeit in %',
         nameLocation: 'center',
         nameGap: 25,
             nameTextStyle: {
@@ -173,7 +208,7 @@ const cCapacityMonthlyOptions = {
     },
     tooltip: sharedOptions.basic.tooltip,
     options: [
-        @foreach ($regionCapacities['region_monthly'] as $m)
+        @foreach ($diagramsOptions['capacityMonthly']['options'] as $m)
         {
         yAxis: {
             data: {!! json_encode($m['months']) !!}
@@ -198,7 +233,7 @@ const cCapacityDaily = echarts.init(chartCapacityDaily);
 const cCapacityDailyOptions = {
     timeline: {
         show: false,
-        data: {!! json_encode($regionCapacities['region']['dates']) !!},
+        data: {!! $diagramsOptions['capacity']['xAxis']['data'] !!},
         axisType: 'time',
     },
     tooltip: sharedOptions.basic.tooltip,
@@ -211,7 +246,7 @@ const cCapacityDailyOptions = {
     xAxis: {
         type: 'value',
         max: 100,
-        name: 'Auslastung in %',
+        name: 'Verfügbarkeit in %',
         nameLocation: 'center',
         nameGap: 25,
         nameTextStyle: {
@@ -222,7 +257,7 @@ const cCapacityDailyOptions = {
         type: 'category',
     },
     options: [
-        @foreach ($regionCapacities['region_daily'] as $d)
+        @foreach ($diagramsOptions['capacityDaily']['options'] as $d)
         {
         yAxis: {
             data: {!! json_encode($d['weekdays']) !!}
@@ -245,10 +280,10 @@ const chartPrediction = document.getElementById('chart-prediction');
 const cPrediction = echarts.init(chartPrediction);
 
 const cPredictionOptions = {
-    color: sharedOptions.basic.color,
+  color: [sharedOptions.basic.color[0], sharedOptions.basic.color[4], sharedOptions.basic.color[5]],
   timeline: {
       show: false,
-      data: {!! json_encode($regionCapacities['region']['dates']) !!},
+      data: {!! $diagramsOptions['capacity']['xAxis']['data'] !!},
       axisType: 'time',
       replaceMerge: ['graphic', 'series']
   },
@@ -266,7 +301,7 @@ const cPredictionOptions = {
   xAxis: {
     type: 'category',
     boundaryGap: false,
-    name: 'Zeitpunkt Scraping',
+    name: 'Zeitpunkt Beobachtung',
     nameLocation: 'center',
     nameGap: 24,
     nameTextStyle: {
@@ -277,7 +312,7 @@ const cPredictionOptions = {
     type: 'value',
     min: 0,
     max: 100,
-    name: 'Auslastung in %',
+    name: 'Verfügbarkeit in %',
     nameLocation: 'center',
     nameGap: 38,
     nameTextStyle: {
@@ -285,7 +320,7 @@ const cPredictionOptions = {
     }
   },
   options: [
-    @foreach ($predictions as $p)
+    @foreach ($diagramsOptions['predictions']['options'] as $p)
     @if($p === null)
     {
       graphic: {
@@ -315,21 +350,27 @@ const cPredictionOptions = {
       series: [
         {
           name: 'Gleitender Mittelwert',
+          showSymbol: false,
+          connectNulls: true,
           type: 'line',
           symbolSize: 7,
-          data: {!! json_encode($p['movAvg']) !!}
+          data: {!! json_encode($p['capacities_moving_average']) !!}
         },
         {
-            name: 'Daten vom ...',
+            name: 'Ausgangsdaten',
+            showSymbol: false,
+            connectNulls: true,
             type: 'line',
             symbolSize: 7,
-            data: {!! json_encode($p['cap_earlierTimeframe']) !!}
+            data: {!! json_encode($p['capacities_timeframe_before']) !!}
         },
         {
-            name: 'Daten vom',
+            name: 'Vergleichsdaten',
+            showSymbol: false,
+            connectNulls: true,
             type: 'line',
             symbolSize: 7,
-            data: {!! json_encode($p['cap_laterTimeframe']) !!}
+            data: {!! json_encode($p['capacities_timeframe_after']) !!}
         }
       ]
     },
@@ -348,10 +389,13 @@ const cHeatmapOptions = {
     position: 'top'
   },
   grid: {
+    show: true,
+    borderWidth: 1,
+    borderColor: '#aaa',
     top: 30,
     right: 45,
-    bottom: 50,
-    left: 5
+    bottom: 70,
+    left: 30
   },
   dataZoom: [{
       type: 'slider'
@@ -362,23 +406,52 @@ const cHeatmapOptions = {
       yAxisIndex: 0,
   }],
   xAxis: {
-    show: false,
-    name: 'Kurzzeitmietobjekt',
+    show: true,
+    name: 'Zeitpunkt Beobachtung',
     type: 'category',
-    data: {!! json_encode($regionPropertiesCapacities['scrapeDates']) !!},
+    data: {!! $diagramsOptions['heatmap']['xAxis']['data'] !!},
+    splitArea: {
+      show: false
+    },
     splitArea: {
       show: false
     },
     axisLabel: {
-      show: true,
+      show: false,
+    },
+    axisTick: {
+        show: false,
+    },
+    axisLine: {
+        show: false,
+    },
+    nameLocation: 'center',
+    nameGap: 10,
+    nameTextStyle: {
+      fontWeight: 'bold',
     }
   },
   yAxis: {
-    show: false,
+    show: true,
     type: 'category',
-    data: {!! json_encode($regionPropertiesCapacities['property_ids']) !!},
+    data: {!! $diagramsOptions['heatmap']['yAxis']['data'] !!},
     splitArea: {
-      show: true
+      show: false
+    },
+    axisTick: {
+        show: false,
+    },
+    axisLine: {
+        show: false,
+    },
+    axisLabel: {
+      show: false,
+    },
+    name: 'Mietobjekte',
+    nameLocation: 'center',
+    nameGap: 10,
+    nameTextStyle: {
+      fontWeight: 'bold',
     }
   },
   visualMap: {
@@ -398,16 +471,16 @@ const cHeatmapOptions = {
   },
   series: [
     {
-      name: 'Auslastung',
+      name: 'Verfügbarkeit',
       type: 'heatmap',
       blurSize: 0,
-      data: {!! json_encode($regionPropertiesCapacities['values']) !!},
+      data: {!! $diagramsOptions['heatmap']['series']['data'] !!},
       label: {
         show: false
       },
       tooltip: {
         formatter: (data) => {
-          return `Kurzzeitmietobjekte-ID: ${data.data[1]}<br />Datum Scraping: ${data.data[0]}<br/>Auslastung: ${data.data[2].toFixed(2)} %`
+          return `Kurzzeitmietobjekte-ID: ${data.data[1]}<br />Beobachtungszeitpunkt: ${data.data[0]}<br/>Verfügbarkeit: ${data.data[2].toFixed(2)} %`
         },
       },
       emphasis: {
@@ -430,7 +503,7 @@ const cTimelineOptions = {
         show: false,
     },
     timeline: {
-        data: {!! json_encode($regionCapacities['region']['dates']) !!},
+        data: {!! $diagramsOptions['capacity']['xAxis']['data'] !!},
         playInterval: 2000,
         axisType: 'time',
         left: 8,

dashboard/routes/console.php

@@ -1,8 +0,0 @@
-<?php
-
-use Illuminate\Foundation\Inspiring;
-use Illuminate\Support\Facades\Artisan;
-
-Artisan::command('inspire', function () {
-    $this->comment(Inspiring::quote());
-})->purpose('Display an inspiring quote')->hourly();

dashboard/routes/web.php

@@ -7,74 +7,124 @@ use App\Chart;
 Route::get('/', function () {
 
     $regionBase = Api::regionBase(-1);
-    $regionBase[] = ['region_name' => 'Alle Regionen', 'region_id' => -1];
-    $regionPropertyCapacities = Api::regionPropertiesCapacities(-1);
+    $regionPropertiesCapacities = Api::regionPropertiesCapacities(-1);
     $propertiesGrowth = Api::propertiesGrowth();
-    $propsPerRegion = Api::propertiesPerRegion();
-    $propsPerRegionName = [];
-    $propsPerRegionCounts = [];
-    $propsPerRegionId = [];
+    $regions = Api::regions()['regions'];
+    $propertiesPerRegion = $regions;
+    $regions[] = ['name' => 'Alle Regionen', 'id' => -1];
+    $propertiesGeo = Api::propertiesGeo()['properties'];
 
-    foreach ($propsPerRegion as $el) {
-        $propsPerRegionName[] = $el['name'];
-        $propsPerRegionId[] = $el['id'];
-        $propsPerRegionCounts[] = $el['count_properties'];
+    $heatmapValues = [];
+
+    foreach ($regionPropertiesCapacities['values'] as $el) {
+        $heatmapValues[] = array_values($el);
     }
 
-    $chartOptions = [
-        'colors' => Chart::colors()
+    $diagramsOptions = [
+        "shared" => [
+            "extractionDates" => json_encode($regionPropertiesCapacities['dates']),
+            "colors" => Chart::colors()
+        ],
+        "heatmap" => [
+            "yAxis" => [
+                "data" => json_encode($regionPropertiesCapacities['property_ids'])
+            ],
+            "series" => [
+                "data" => json_encode($heatmapValues)
+            ]
+        ],
+        "propertiesPerRegion" => [
+            "yAxis" => [
+                "data" => json_encode(array_column($propertiesPerRegion, 'count_properties'))
+            ],
+            "xAxis" => [
+                "data" => json_encode(array_column($propertiesPerRegion, 'name'))
+            ]
+        ],
+        "extractions" => [
+            "series" => $propertiesGrowth,
+        ]
     ];
 
-    $propertiesGeo = Api::propertiesGeo();
-
     return view('overview', [
-        "regions" => $regionBase,
-        "regionPropertiesCapacities" => $regionPropertyCapacities,
+        "regions" => $regions,
+        "region" => $regionBase,
+        "diagramsOptions" => $diagramsOptions,
         "geo" => $propertiesGeo,
-        "growth" => $propertiesGrowth,
-        "chartOptions" => $chartOptions,
-        "propsPerRegion" => [json_encode($propsPerRegionId), json_encode($propsPerRegionName), json_encode($propsPerRegionCounts)]]);
+
+    ]);
 });
 
 Route::get('/region/{id}', function (int $id) {
 
-    $regionBaseAll = Api::regionBase(-1);
-    $regionBaseAll[] = ['region_name' => 'Alle Regionen', 'region_id' => -1];
-    $regionBaseRegion = $id >= 0 ? Api::regionBase($id) : [['region_name' => 'Alle Regionen']];
-
+    $regions = Api::regions()['regions'];
+    $regions[] = ['name' => 'Alle Regionen', 'id' => -1];
+    $region = $id >= 0 ? Api::regionBase($id) : ['name' => 'Alle Regionen'];
     $regionPropertiesCapacities = Api::regionPropertiesCapacities($id);
     $regionCapacitiesRegion = Api::regionCapacities($id);
     $regionCapacitiesAll = Api::regionCapacities(-1);
-
     $regionCapacitiesMonthly = [];
     $regionCapacitiesDaily = [];
     $regionPredictions = [];
 
+    $heatmapValues = [];
+
+    foreach ($regionPropertiesCapacities['values'] as $el) {
+        $heatmapValues[] = array_values($el);
+    }
+
     foreach ($regionCapacitiesRegion['dates'] as $date) {
         $regionCapacitiesMonthly[] = Api::regionCapacitiesMonthly($id, $date);
         $regionCapacitiesDaily[] = Api::regionCapacitiesDaily($id, $date);
         $regionPredictions[] = Api::regionMovingAverage($id, $date);
     }
 
-    $chartOptions = [
-        'colors' => Chart::colors()
-    ];
-
-    $regionCapacities = [
-        'all' => $regionCapacitiesAll,
-        'region' => $regionCapacitiesRegion,
-        'region_monthly' => $regionCapacitiesMonthly,
-        'region_daily' => $regionCapacitiesDaily
-
+    $diagramsOptions = [
+        "shared" => [
+            "extractionDates" => json_encode($regionPropertiesCapacities['dates']),
+            "colors" => Chart::colors()
+        ],
+        "heatmap" => [
+            "xAxis" => [
+                "data" => json_encode($regionPropertiesCapacities['dates'])
+            ],
+            "yAxis" => [
+                "data" => json_encode($regionPropertiesCapacities['property_ids'])
+            ],
+            "series" => [
+                "data" => json_encode($heatmapValues)
+            ]
+        ],
+        "predictions" => [
+            "options" => $regionPredictions,
+        ],
+        "capacityMonthly" => [
+            "options" => $regionCapacitiesMonthly,
+        ],
+        "capacityDaily" => [
+            "options" => $regionCapacitiesDaily,
+        ],
+        "capacity" => [
+            "xAxis" => [
+                "data" => json_encode($regionCapacitiesRegion['dates'])
+            ],
+            "series" => [
+                "all" => [
+                    "data" => json_encode($regionCapacitiesAll['capacities'])
+                ],
+                "region" => [
+                    "data" =>  json_encode($regionCapacitiesRegion['capacities'])
+                ]
+            ]
+        ]
     ];
 
     return view('region', [
-        'chartOptions' => $chartOptions,
+        'diagramsOptions' => $diagramsOptions,
         'startDate' => $regionCapacitiesRegion['dates'][0],
-        'regions' => $regionBaseAll,
-        'region' => $regionBaseRegion,
-        'region_id' => $id,
-        'regionCapacities' => $regionCapacities,
+        'regions' => $regions,
+        'region' => $region,
+        'regionId' => $id,
         'regionPropertiesCapacities' => $regionPropertiesCapacities,
         'predictions' => $regionPredictions]);
 
@@ -83,43 +133,48 @@ Route::get('/region/{id}', function (int $id) {
 
 Route::get('/property/{id}', function (int $id) {
 
-    $chartOptions = [
-        'colors' => Chart::colors()
-    ];
-    $regionBaseAll = Api::regionBase(-1);
-    $regionBaseAll[] = ['region_name' => 'Alle Regionen', 'region_id' => -1];
-    $propertyBase = Api::propertyBase($id);
-    $calendars = Api::propertyExtractions($id);
+
+    $regions = Api::regions()['regions'];
+    $regions[] = ['name' => 'Alle Regionen', 'id' => -1];
+
+    $base = Api::propertyBase($id);
+    $calendars = Api::propertyExtractions($id)['extractions'];
     $propertyCapacities = Api::propertyCapacities($id);
-    $propertyNeighbours = Api::propertyNeighbours($id);
-    $regionCapacitiesRegion = Api::regionCapacities($propertyBase[0]['region_id']);
+    $propertyNeighbours = Api::propertyNeighbours($id)['neighbours'];
+    $regionCapacitiesRegion = Api::regionCapacities($base['region_id']);
     $regionCapacitiesAll = Api::regionCapacities(-1);
     $regionCapacities = [[],[]];
 
     $propertyCapacitiesMonthly = [];
     $propertyCapacitiesDaily = [];
 
-    foreach ($propertyCapacities['dates'] as $date) {
-        $propertyCapacitiesMonthly[] = Api::propertyCapacitiesMonthly($id, $date);
-        $propertyCapacitiesDaily[] = Api::propertyCapacitiesDaily($id, $date);
-    }
-
-    // filter out all date, which were not scraped for the property
-    foreach ($regionCapacitiesAll['dates'] as $index => $date) {
-        if(in_array($date, $propertyCapacities['dates'])){
-            $regionCapacities[0][] = $regionCapacitiesAll['capacities'][$index];
+    if($propertyCapacities){
+        foreach ($propertyCapacities['dates'] as $date) {
+            $propertyCapacitiesMonthly[] = Api::propertyCapacitiesMonthly($id, $date);
+            $propertyCapacitiesDaily[] = Api::propertyCapacitiesDaily($id, $date);
         }
-    }
 
-    foreach ($regionCapacitiesRegion['dates'] as $index => $date) {
-        if(in_array($date, $propertyCapacities['dates'])){
-            $regionCapacities[1][] = $regionCapacitiesRegion['capacities'][$index];
+        // filter out all date, which were not scraped for the property
+        foreach ($regionCapacitiesAll['dates'] as $index => $date) {
+            if(in_array($date, $propertyCapacities['dates'])){
+                $regionCapacities[0][] = $regionCapacitiesAll['capacities'][$index];
+            }
         }
+
+        foreach ($regionCapacitiesRegion['dates'] as $index => $date) {
+            if(in_array($date, $propertyCapacities['dates'])){
+                $regionCapacities[1][] = $regionCapacitiesRegion['capacities'][$index];
+            }
+        }
+    }else{
+        return view('property-nodata', [
+            'base' => $base,
+            'regions' => $regions,
+        ]);
     }
 
     // prepare data for calendar chart
-    $data = [];
-    $dates = [];
+    $calendarData = [];
 
     foreach ($calendars as $el) {
 
@@ -130,21 +185,41 @@ Route::get('/property/{id}', function (int $id) {
             $series[] = [$date, $status];
         }
 
-        $data[] = $series;
+        $calendarData[] = $series;
 
     }
 
+    $diagramsOptions = [
+        "shared" => [
+            "colors" => Chart::colors(),
+            "extractiondates" => json_encode($propertyCapacities['dates'])
+        ],
+        "calendar" => [
+            "series" => $calendarData
+        ],
+        "capacities" => [
+            "xAxis" => [
+                "data" => json_encode($propertyCapacities['dates'])
+            ],
+            "series" => [
+                ["data" => json_encode($propertyCapacities['capacities'])],
+                ["data" => json_encode($regionCapacities[0])],
+                ["data" => json_encode($regionCapacities[1])],
+            ]
+        ],
+        "capacityMonthly" => [
+            "options" => $propertyCapacitiesMonthly,
+        ],
+        "capacityDaily" => [
+            "options" => $propertyCapacitiesDaily,
+        ],
+    ];
+
     return view('property', [
-        'chartOptions' => $chartOptions,
+        'diagramsOptions' => $diagramsOptions,
         'startDate' => $propertyCapacities['dates'][0],
-        'base' => $propertyBase[0],
-        'regions' => $regionBaseAll,
-        'extractiondates' => json_encode($propertyCapacities['dates']),
-        'calendar' => $data,
-        'propertyCapacities' => $propertyCapacities,
-        'capacitiesMonthly' => $propertyCapacitiesMonthly,
-        'capacitiesDaily' => $propertyCapacitiesDaily,
-        'regionCapacities' => $regionCapacities,
+        'base' => $base,
+        'regions' => $regions,
         'neighbours' => $propertyNeighbours
     ]);
 

documentation/diagrams/ETL_C4.drawio

@@ -1,117 +1,141 @@
-<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36" version="26.0.5">
+<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0" version="26.0.6">
   <diagram name="Seite-1" id="WNMV8rePnVf-2Vz_xhjt">
-    <mxGraphModel dx="1688" dy="1050" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+    <mxGraphModel dx="1937" dy="1185" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
       <root>
         <mxCell id="0" />
         <mxCell id="1" parent="0" />
+        <mxCell id="e6qn9whkbaCBCFCjUvdY-7" value="" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=none;fillColor=#F5F5F5;" vertex="1" parent="1">
+          <mxGeometry x="10" y="420" width="1070" height="690" as="geometry" />
+        </mxCell>
+        <object placeholders="1" c4Name="ETL" c4Type="ContainerScopeBoundary" c4Application="Component" label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;&lt;div style=&quot;text-align: left&quot;&gt;%c4Name%&lt;/div&gt;&lt;/b&gt;&lt;/font&gt;&lt;div style=&quot;text-align: left&quot;&gt;[%c4Application%]&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-11">
+          <mxCell style="rounded=1;fontSize=11;whiteSpace=wrap;html=1;dashed=1;arcSize=20;fillColor=default;strokeColor=#666666;fontColor=#333333;labelBackgroundColor=none;align=left;verticalAlign=bottom;labelBorderColor=none;spacingTop=0;spacing=10;dashPattern=8 4;metaEdit=1;rotatable=0;perimeter=rectanglePerimeter;noLabel=0;labelPadding=0;allowArrows=0;connectable=0;expand=0;recursiveResize=0;editable=1;pointerEvents=0;absoluteArcSize=1;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
+            <mxGeometry x="30" y="440" width="1030" height="500" as="geometry" />
+          </mxCell>
+        </object>
         <object placeholders="1" c4Name="Datenbank Aggregation" c4Type="Container" c4Technology="MySQL" c4Description="Datenbank welche während Aggregation verwendet wurde." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-1">
-          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" vertex="1" parent="1">
+          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" parent="1" vertex="1">
             <mxGeometry x="40" y="100" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <object placeholders="1" c4Name="Datenbank Analyse" c4Type="Container" c4Technology="DuckDB" c4Description="Datenbank, welcher für die Analysen verwendet wurden." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-2">
-          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" vertex="1" parent="1">
+        <object placeholders="1" c4Name="Datenbank Analyse" c4Type="Container" c4Technology="DuckDB" c4Description="Datenbank, welcher für die Analysen&lt;br&gt; verwendet wurden." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-2">
+          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" parent="1" vertex="1">
             <mxGeometry x="790" y="100" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-5" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="0Mexl9jQAquWokRCgHYt-3" target="0Mexl9jQAquWokRCgHYt-1">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-5" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;dashed=1;dashPattern=8 8;" parent="1" source="0Mexl9jQAquWokRCgHYt-3" target="0Mexl9jQAquWokRCgHYt-1" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-7" value="liest Datenbank" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="0Mexl9jQAquWokRCgHYt-5">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-7" value="liest Datenbank" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0Mexl9jQAquWokRCgHYt-5" vertex="1" connectable="0">
           <mxGeometry x="-0.2497" y="-1" relative="1" as="geometry">
             <mxPoint x="-10" y="1" as="offset" />
           </mxGeometry>
         </mxCell>
         <object placeholders="1" c4Name="Sling" c4Type="sling-cli" c4Description="Kommandozeilenprogramm zur Migration von Datensätzen." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#cccccc&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-3">
-          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" vertex="1" parent="1">
+          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
             <mxGeometry x="400" y="100" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-6" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="0Mexl9jQAquWokRCgHYt-3" target="0Mexl9jQAquWokRCgHYt-2">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-6" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" parent="1" source="0Mexl9jQAquWokRCgHYt-3" target="0Mexl9jQAquWokRCgHYt-2" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-8" value="schreibt in Datenbank" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="0Mexl9jQAquWokRCgHYt-6">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-8" value="schreibt in Datenbank" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0Mexl9jQAquWokRCgHYt-6" vertex="1" connectable="0">
           <mxGeometry x="-0.1744" relative="1" as="geometry">
-            <mxPoint x="16" y="-1" as="offset" />
+            <mxPoint x="12" y="-1" as="offset" />
           </mxGeometry>
         </mxCell>
         <object placeholders="1" c4Name="Preprocessing" c4Type="ContainerScopeBoundary" c4Application="Component" label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;&lt;div style=&quot;text-align: left&quot;&gt;%c4Name%&lt;/div&gt;&lt;/b&gt;&lt;/font&gt;&lt;div style=&quot;text-align: left&quot;&gt;[%c4Application%]&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-9">
-          <mxCell style="rounded=1;fontSize=11;whiteSpace=wrap;html=1;dashed=1;arcSize=20;fillColor=none;strokeColor=#666666;fontColor=#333333;labelBackgroundColor=none;align=left;verticalAlign=bottom;labelBorderColor=none;spacingTop=0;spacing=10;dashPattern=8 4;metaEdit=1;rotatable=0;perimeter=rectanglePerimeter;noLabel=0;labelPadding=0;allowArrows=0;connectable=0;expand=0;recursiveResize=0;editable=1;pointerEvents=0;absoluteArcSize=1;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" vertex="1" parent="1">
+          <mxCell style="rounded=1;fontSize=11;whiteSpace=wrap;html=1;dashed=1;arcSize=20;fillColor=none;strokeColor=#666666;fontColor=#333333;labelBackgroundColor=none;align=left;verticalAlign=bottom;labelBorderColor=none;spacingTop=0;spacing=10;dashPattern=8 4;metaEdit=1;rotatable=0;perimeter=rectanglePerimeter;noLabel=0;labelPadding=0;allowArrows=0;connectable=0;expand=0;recursiveResize=0;editable=1;pointerEvents=0;absoluteArcSize=1;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
             <mxGeometry x="20" y="40" width="1030" height="270" as="geometry" />
           </mxCell>
         </object>
-        <object placeholders="1" c4Name="Datenbank Analyse" c4Type="Container" c4Technology="DuckDB" c4Description="Datenbank, welcher für die Analysen verwendet wurden." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-10">
-          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" vertex="1" parent="1">
+        <object placeholders="1" c4Name="Datenbank" c4Type="Container" c4Technology="DuckDB" c4Description="Datenbank, welcher für die Analysen&lt;br&gt; verwendet wurden." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-10">
+          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" parent="1" vertex="1">
             <mxGeometry x="80" y="480" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <object placeholders="1" c4Name="Datenaufbereitung" c4Type="ContainerScopeBoundary" c4Application="ETL" label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;&lt;div style=&quot;text-align: left&quot;&gt;%c4Name%&lt;/div&gt;&lt;/b&gt;&lt;/font&gt;&lt;div style=&quot;text-align: left&quot;&gt;[%c4Application%]&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-11">
-          <mxCell style="rounded=1;fontSize=11;whiteSpace=wrap;html=1;dashed=1;arcSize=20;fillColor=none;strokeColor=#666666;fontColor=#333333;labelBackgroundColor=none;align=left;verticalAlign=bottom;labelBorderColor=none;spacingTop=0;spacing=10;dashPattern=8 4;metaEdit=1;rotatable=0;perimeter=rectanglePerimeter;noLabel=0;labelPadding=0;allowArrows=0;connectable=0;expand=0;recursiveResize=0;editable=1;pointerEvents=0;absoluteArcSize=1;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" vertex="1" parent="1">
-            <mxGeometry x="30" y="440" width="1030" height="490" as="geometry" />
-          </mxCell>
-        </object>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-23" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="0Mexl9jQAquWokRCgHYt-12" target="0Mexl9jQAquWokRCgHYt-14">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-23" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" parent="1" source="0Mexl9jQAquWokRCgHYt-12" target="0Mexl9jQAquWokRCgHYt-14" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-24" value="verwendet" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="0Mexl9jQAquWokRCgHYt-23">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-24" value="verwendet" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontSize=14;" parent="0Mexl9jQAquWokRCgHYt-23" vertex="1" connectable="0">
           <mxGeometry x="-0.0114" y="-2" relative="1" as="geometry">
-            <mxPoint as="offset" />
+            <mxPoint y="-2" as="offset" />
           </mxGeometry>
         </mxCell>
         <object placeholders="1" c4Name="etl_*.py" c4Type="Python (Polars)" c4Description="Diverse Python Skripts zur Aufbereitung / Zusammenstellung der Daten." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#cccccc&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-12">
-          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" vertex="1" parent="1">
+          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
             <mxGeometry x="430" y="710" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="0Mexl9jQAquWokRCgHYt-13" target="0Mexl9jQAquWokRCgHYt-10">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;dashed=1;dashPattern=8 8;" parent="1" source="0Mexl9jQAquWokRCgHYt-13" target="0Mexl9jQAquWokRCgHYt-10" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-17" value="Liest Datenbank" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="0Mexl9jQAquWokRCgHYt-16">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-17" value="liest Datenbank" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontSize=14;" parent="0Mexl9jQAquWokRCgHYt-16" vertex="1" connectable="0">
           <mxGeometry x="-0.1633" relative="1" as="geometry">
             <mxPoint as="offset" />
           </mxGeometry>
         </mxCell>
         <object placeholders="1" c4Name="database.py" c4Type="Python (DuckDB Interface)" c4Description="Wrapper Skript zum Ausführen von SQL." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#cccccc&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-13">
-          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" vertex="1" parent="1">
+          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
             <mxGeometry x="80" y="710" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-18" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;dashed=1;dashPattern=8 8;entryX=0.24;entryY=0.981;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="0Mexl9jQAquWokRCgHYt-14" target="0Mexl9jQAquWokRCgHYt-15">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-18" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;dashed=1;dashPattern=8 8;entryX=0.24;entryY=0.981;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="0Mexl9jQAquWokRCgHYt-14" target="0Mexl9jQAquWokRCgHYt-15" edge="1">
           <mxGeometry relative="1" as="geometry">
             <mxPoint x="900" y="600" as="targetPoint" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-19" value="schreibt pickle objekt" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="0Mexl9jQAquWokRCgHYt-18">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-19" value="schreibt pickle objekt" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontSize=14;" parent="0Mexl9jQAquWokRCgHYt-18" vertex="1" connectable="0">
           <mxGeometry x="-0.1818" y="2" relative="1" as="geometry">
             <mxPoint as="offset" />
           </mxGeometry>
         </mxCell>
         <object placeholders="1" c4Name="etl_cache.py" c4Type="Python (Pickle)" c4Description="Diverse Python Skripts zur Aufbereitung / Zusammenstellung der Daten." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#cccccc&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-14">
-          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" vertex="1" parent="1">
+          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
             <mxGeometry x="780" y="710" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <object placeholders="1" c4Name="Cache" c4Type="Container" c4Technology="Filesystem" c4Description="Das Dateisystem wird als Pufferspeicher verwendet. Die Daten werden als Pickle Objekte gespeichert." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-15">
-          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" vertex="1" parent="1">
+        <object placeholders="1" c4Name="Cache" c4Type="Container" c4Technology="Filesystem" c4Description="Das Dateisystem wird als Pufferspeicher verwendet." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="0Mexl9jQAquWokRCgHYt-15">
+          <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" parent="1" vertex="1">
             <mxGeometry x="780" y="480" width="240" height="120" as="geometry" />
           </mxCell>
         </object>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-20" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.746;entryY=1.002;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;exitX=0.75;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;" edge="1" parent="1" source="0Mexl9jQAquWokRCgHYt-14" target="0Mexl9jQAquWokRCgHYt-15">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-20" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.746;entryY=1.002;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;exitX=0.75;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;" parent="1" source="0Mexl9jQAquWokRCgHYt-14" target="0Mexl9jQAquWokRCgHYt-15" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-21" value="liest pickle objekt" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="0Mexl9jQAquWokRCgHYt-20">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-21" value="liest pickle objekt" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontSize=14;" parent="0Mexl9jQAquWokRCgHYt-20" vertex="1" connectable="0">
           <mxGeometry x="-0.1076" y="1" relative="1" as="geometry">
             <mxPoint x="8" y="-11" as="offset" />
           </mxGeometry>
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-25" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="0Mexl9jQAquWokRCgHYt-12" target="0Mexl9jQAquWokRCgHYt-13">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-25" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" parent="1" source="0Mexl9jQAquWokRCgHYt-12" target="0Mexl9jQAquWokRCgHYt-13" edge="1">
           <mxGeometry relative="1" as="geometry" />
         </mxCell>
-        <mxCell id="0Mexl9jQAquWokRCgHYt-26" value="Verwendet" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="0Mexl9jQAquWokRCgHYt-25">
+        <mxCell id="0Mexl9jQAquWokRCgHYt-26" value="verwendet" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontSize=14;" parent="0Mexl9jQAquWokRCgHYt-25" vertex="1" connectable="0">
           <mxGeometry x="0.0473" relative="1" as="geometry">
             <mxPoint as="offset" />
           </mxGeometry>
         </mxCell>
+        <mxCell id="e6qn9whkbaCBCFCjUvdY-3" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="e6qn9whkbaCBCFCjUvdY-1" target="0Mexl9jQAquWokRCgHYt-13">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="e6qn9whkbaCBCFCjUvdY-6" value="führt Funktionen aus" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontSize=14;" vertex="1" connectable="0" parent="e6qn9whkbaCBCFCjUvdY-3">
+          <mxGeometry x="0.0906" relative="1" as="geometry">
+            <mxPoint y="1" as="offset" />
+          </mxGeometry>
+        </mxCell>
+        <object placeholders="1" c4Name="FastAPI" c4Type="Python (FastAPI)" c4Description="Stellt die aufbereiteten Daten über eine JSON-Schnittstelle zur Verfügung." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#cccccc&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="e6qn9whkbaCBCFCjUvdY-1">
+          <mxCell style="rounded=1;whiteSpace=wrap;html=1;labelBackgroundColor=none;fillColor=#1061B0;fontColor=#ffffff;align=center;arcSize=10;strokeColor=#0D5091;metaEdit=1;resizable=0;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" vertex="1" parent="1">
+            <mxGeometry x="430" y="970" width="240" height="120" as="geometry" />
+          </mxCell>
+        </object>
+        <mxCell id="e6qn9whkbaCBCFCjUvdY-2" style="rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="e6qn9whkbaCBCFCjUvdY-1" target="0Mexl9jQAquWokRCgHYt-12">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="e6qn9whkbaCBCFCjUvdY-5" value="führt Funktionen aus" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];fontSize=14;" vertex="1" connectable="0" parent="e6qn9whkbaCBCFCjUvdY-2">
+          <mxGeometry x="0.0286" y="-1" relative="1" as="geometry">
+            <mxPoint as="offset" />
+          </mxGeometry>
+        </mxCell>
       </root>
     </mxGraphModel>
   </diagram>

documentation/diagrams/Systemarchitektur_C4.drawio

@@ -1,10 +1,18 @@
-<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36" version="26.0.5" pages="2">
+<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0" version="26.0.6" pages="2">
   <diagram name="Seite-1" id="chpUGVRRn7alPJZ1I-il">
-    <mxGraphModel dx="2761" dy="1531" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+    <mxGraphModel dx="1291" dy="790" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
       <root>
         <mxCell id="0" />
         <mxCell id="1" parent="0" />
-        <object placeholders="1" c4Name="RDBMS" c4Type="Container" c4Technology="DuckDB" c4Description="Aggregierte Daten." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="_wAeSdXpbb6KPP4DEc36-2">
+        <mxCell id="tzVNFCieMdwak3VSEkXc-1" value="" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=none;fillColor=#F5F5F5;" vertex="1" parent="1">
+          <mxGeometry x="10" y="20" width="750" height="780" as="geometry" />
+        </mxCell>
+        <object placeholders="1" c4Name="Visual Analytics Tool" c4Type="SystemScopeBoundary" c4Application="Software System" label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;&lt;div style=&quot;text-align: left&quot;&gt;%c4Name%&lt;/div&gt;&lt;/b&gt;&lt;/font&gt;&lt;div style=&quot;text-align: left&quot;&gt;[%c4Application%]&lt;/div&gt;" id="_wAeSdXpbb6KPP4DEc36-23">
+          <mxCell style="rounded=1;fontSize=11;whiteSpace=wrap;html=1;dashed=1;arcSize=20;fillColor=default;strokeColor=#666666;fontColor=#333333;labelBackgroundColor=none;align=left;verticalAlign=bottom;labelBorderColor=none;spacingTop=0;spacing=10;dashPattern=8 4;metaEdit=1;rotatable=0;perimeter=rectanglePerimeter;noLabel=0;labelPadding=0;allowArrows=0;connectable=0;expand=0;recursiveResize=0;editable=1;pointerEvents=0;absoluteArcSize=1;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
+            <mxGeometry x="30" y="40" width="710" height="540" as="geometry" />
+          </mxCell>
+        </object>
+        <object placeholders="1" c4Name="Datenbank" c4Type="Container" c4Technology="DuckDB" c4Description="Aggregierte Daten." label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;%c4Name%&lt;/b&gt;&lt;/font&gt;&lt;div&gt;[%c4Type%:&amp;nbsp;%c4Technology%]&lt;/div&gt;&lt;br&gt;&lt;div&gt;&lt;font style=&quot;font-size: 11px&quot;&gt;&lt;font color=&quot;#E6E6E6&quot;&gt;%c4Description%&lt;/font&gt;&lt;/div&gt;" id="_wAeSdXpbb6KPP4DEc36-2">
           <mxCell style="shape=cylinder3;size=15;whiteSpace=wrap;html=1;boundedLbl=1;rounded=0;labelBackgroundColor=none;fillColor=#23A2D9;fontSize=12;fontColor=#ffffff;align=center;strokeColor=#0E7DAD;metaEdit=1;points=[[0.5,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.5,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];resizable=0;" parent="1" vertex="1">
             <mxGeometry x="50" y="60" width="240" height="120" as="geometry" />
           </mxCell>
@@ -82,11 +90,6 @@
             <mxPoint x="5" y="7" as="offset" />
           </mxGeometry>
         </mxCell>
-        <object placeholders="1" c4Name="Visual Analytics Tool" c4Type="SystemScopeBoundary" c4Application="Software System" label="&lt;font style=&quot;font-size: 16px&quot;&gt;&lt;b&gt;&lt;div style=&quot;text-align: left&quot;&gt;%c4Name%&lt;/div&gt;&lt;/b&gt;&lt;/font&gt;&lt;div style=&quot;text-align: left&quot;&gt;[%c4Application%]&lt;/div&gt;" id="_wAeSdXpbb6KPP4DEc36-23">
-          <mxCell style="rounded=1;fontSize=11;whiteSpace=wrap;html=1;dashed=1;arcSize=20;fillColor=none;strokeColor=#666666;fontColor=#333333;labelBackgroundColor=none;align=left;verticalAlign=bottom;labelBorderColor=none;spacingTop=0;spacing=10;dashPattern=8 4;metaEdit=1;rotatable=0;perimeter=rectanglePerimeter;noLabel=0;labelPadding=0;allowArrows=0;connectable=0;expand=0;recursiveResize=0;editable=1;pointerEvents=0;absoluteArcSize=1;points=[[0.25,0,0],[0.5,0,0],[0.75,0,0],[1,0.25,0],[1,0.5,0],[1,0.75,0],[0.75,1,0],[0.5,1,0],[0.25,1,0],[0,0.75,0],[0,0.5,0],[0,0.25,0]];" parent="1" vertex="1">
-            <mxGeometry x="30" y="40" width="710" height="540" as="geometry" />
-          </mxCell>
-        </object>
       </root>
     </mxGraphModel>
   </diagram>

etl/README.md

@@ -1,4 +1,25 @@
-# How to run
+## Installation
+Folgende Schritte zur Installation vornehmen
+
+### Abhängigkeiten installieren
+Zur Verwaltung der Abhängigkeiten wird [pixi](https://pixi.sh/) verwendet.
+```bash
+pixi install
+```
+
+### Datenbankverbindung konfigurieren
+Enviroment File erstellen:
+```bash
+cp src/.env.example .env
+```
+
+Im erstellten .env File die Datei anpassen:
+```
+DATABASE="/path/to/db.duckdb"
+```
+
+# FastAPI starten
+FastAPI auf einem anderen Port ausführen als das Dashboard.
 ```bash
 fastapi dev api/main.py --port 8080
 ```

etl/pixi.lock

@@ -2196,13 +2196,6 @@ packages:
   - pkg:pypi/colorama?source=hash-mapping
   size: 25170
   timestamp: 1666700778190
-- kind: pypi
-  name: consultancy-2
-  version: 0.1.0
-  path: .
-  sha256: c09f63486f0dd4151008de68ef73d00f72663dc3cc47894ff750d517f898a23b
-  requires_python: '>=3.11'
-  editable: true
 - kind: conda
   name: dnspython
   version: 2.7.0
@@ -2280,6 +2273,13 @@ packages:
   purls: []
   size: 6690
   timestamp: 1718984720419
+- kind: pypi
+  name: etl
+  version: 0.1.0
+  path: .
+  sha256: d682071d587e9be1fcf91237a1add69a92c34715bc491a4067b07d63ce79616d
+  requires_python: '>=3.11'
+  editable: true
 - kind: conda
   name: exceptiongroup
   version: 1.2.2

etl/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 authors = [{name = "Giò Diani", email = "mail@gionathandiani.name"}, {name = "Mauro Stoffel", email = "mauro.stoffel@stud.fhgr.ch"}, {name = "Colin Bolli", email = "colin.bolli@stud.fhgr.ch"}, {name = "Charles Winkler", email = "charles.winkler@stud.fhgr.ch"}]
 description = "Datenauferbeitung"
-name = "consultancy_2"
+name = "ETL"
 requires-python = ">= 3.11"
 version = "0.1.0"
 
@@ -14,7 +14,7 @@ channels = ["conda-forge"]
 platforms = ["win-64", "linux-64", "osx-64", "osx-arm64"]
 
 [tool.pixi.pypi-dependencies]
-consultancy_2 = { path = ".", editable = true }
+etl = { path = ".", editable = true }
 
 [tool.pixi.tasks]
 

etl/src/api/main.py

@@ -1,104 +1,268 @@
+import datetime
+from typing import List, Union
+
 import data
 import polars as pl
 from data import etl_property_capacities as etl_pc
+from data import etl_property_capacities_daily as etl_pcd
 from data import etl_property_capacities_monthly as etl_pcm
-from data import etl_property_capacities_weekdays as etl_pcw
 from data import etl_property_neighbours as etl_pn
 from data import etl_region_capacities as etl_rc
-from data import etl_region_capacities_comparison as etl_rcc
+from data import etl_region_capacities_daily as etl_rcd
 from data import etl_region_capacities_monthly as etl_rcm
-from data import etl_region_capacities_weekdays as etl_rcw
 from data import etl_region_movAverage as etl_rmA
 from data import etl_region_properties_capacities as etl_rpc
-from fastapi import FastAPI, Response
+from fastapi import FastAPI
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+
+
+class RegionsItems(BaseModel):
+    name: str
+    id: str
+    count_properties: int
+
+class Regions(BaseModel):
+    regions: List[RegionsItems]
+
+class RegionBase(BaseModel):
+    name: str
+    id: str
+
+class RegionPropertiesCapacitiesValues(BaseModel):
+	date: str
+	property_id: str
+	capacity: float
+
+class RegionCapacities(BaseModel):
+    capacities: List[float]
+    dates: List
+
+class RegionCapacitiesMonthly(BaseModel):
+    months: List[str]
+    capacities: List[float]
+
+class RegionCapacitiesDaily(BaseModel):
+    weekdays: List[str]
+    capacities: List[float]
+
+class RegionPropertiesCapacities(BaseModel):
+    dates: List
+    property_ids: List
+    values: List[RegionPropertiesCapacitiesValues]
+
+class RegionMovingAverage(BaseModel):
+    dates: List
+    capacities_timeframe_before: List[Union[float, None]]
+    capacities_timeframe_after: List[Union[float, None]]
+    capacities_moving_average: List[Union[float, None]]
+
+class PropertiesGrowth(BaseModel):
+    dates: List
+    total_all: List[Union[int, None]]
+    total_heidiland: List[Union[int, None]]
+    total_engadin: List[Union[int, None]]
+    total_stmoritz: List[Union[int, None]]
+    total_davos: List[Union[int, None]]
+
+class PropertiesGeoList(BaseModel):
+    property_id: str
+    latlng: str
+    region_id: str
+
+class PropertiesGeo(BaseModel):
+    properties: List[PropertiesGeoList]
+
+class PropertyNeighboursList(BaseModel):
+    id: str
+    lat: float
+    lon: float
+
+class PropertyNeighbours(BaseModel):
+    neighbours: List[PropertyNeighboursList]
+
+class PropertyNeighboursList(BaseModel):
+    id: str
+    lat: float
+    lon: float
+
+class PropertyExtractionsList(BaseModel):
+    calendar: str
+    date: str
+
+class PropertyExtractions(BaseModel):
+    extractions: List[PropertyExtractionsList]
+
+class PropertyCapacities(BaseModel):
+    capacities: List[float]
+    dates: List[str]
+
+class PropertyCapacitiesMonthly(BaseModel):
+    months: List[str]
+    capacities: List[float]
+
+class PropertyCapacitiesDaily(BaseModel):
+    weekdays: List[str]
+    capacities: List[float]
+
+class PropertyBaseDetail(BaseModel):
+    property_platform_id: str
+    first_found: str
+    last_found: str
+    latlng: str
+    region_id: str
+    region_name: str
+
+class PropertyBase(BaseModel):
+    property_platform_id: str
+    first_found: str
+    last_found: str
+    latlng: str
+    region_id: str
+    region_name: str
+
 
 d = data.load()
 
-app = FastAPI()
+tags_metadata = [
+    {
+        "name": "regions",
+        "description": "Get data for regions.",
+    },
+    {
+        "name": "properties",
+        "description": "Get data for properties",
+    },
+]
+
+app = FastAPI(openapi_tags=tags_metadata)
 
 @app.get("/")
 def read_root():
     return {"Hi there!"}
 
-@app.get("/items/{item_id}")
-def read_item(item_id: int):
-	ext = d.extractions_for(item_id).pl()
-	out = ext.with_columns(pl.col("calendar").str.extract_all(r"([0-9]{4}-[0-9]{2}-[0-9]{2})|[0-2]").alias("calendar_data"))
-	out = out.drop(['calendar', 'property_id'])
-	return Response(content=out.write_json(), media_type="application/json")
+@app.get("/regions", response_model=Regions, tags=['region'])
+def regions():
+	"""
+	Returns a list of all available regions.
+	"""
+	return {"regions" : d.properties_per_region().pl().to_dicts()}
 
-@app.get("/region/properties")
-def properties_region():
-	return d.properties_per_region().pl().to_dicts()
+@app.get("/regions/{id}/base", response_model=RegionBase, tags=['region'])
+def region_base(id: int):
+	"""
+	Returns basic information about a region.
+	"""
+	base = d.region_base_data(id).pl().to_dicts()
+	return {"id": base[0]["id"], "name": base[0]["name"]}
 
-@app.get("/properties/growth")
-def properties_growth():
-	options = {"dates" : d.properties_growth().pl()['date'].to_list(), "total_all" : d.properties_growth().pl()['total_all'].to_list(),  "total_heidiland" : d.properties_growth().pl()['total_heidiland'].to_list(), "total_engadin" : d.properties_growth().pl()['total_engadin'].to_list(), "total_davos" : d.properties_growth().pl()['total_davos'].to_list(), "total_stmoritz" : d.properties_growth().pl()['total_stmoritz'].to_list()}
-	return options
-
-@app.get("/properties/geo")
-def properties_geo():
-	return d.properties_geo().pl().to_dicts()
-
-@app.get("/property/{id}/neighbours")
-def property_neighbours(id: int):
-	capacities = etl_pn.property_neighbours(id)
-	return capacities
-
-@app.get("/property/{id}/extractions")
-def property_extractions(id: int):
-	return d.extractions_for(property_id = id).pl().to_dicts()
-
-@app.get("/property/{id}/capacities")
-def property_capacities_data(id: int):
-	capacities = etl_pc.property_capacities(id)
-	return capacities
-
-@app.get("/property/{id}/capacities/monthly/{scrapeDate}")
-def property_capacities_data(id: int, scrapeDate: str):
-	capacities = etl_pcm.property_capacities_monthly(id, scrapeDate)
-	return capacities
-
-@app.get("/property/{id}/capacities/weekdays/{scrapeDate}")
-def property_capacities_data(id: int, scrapeDate: str):
-	capacities = etl_pcw.property_capacities_weekdays(id, scrapeDate)
-	return capacities
-
-@app.get("/property/{id}/base")
-def property_base_data(id: int):
-	return d.property_base_data(id).pl().to_dicts()
-
-@app.get("/region/{id}/properties/capacities")
-def region_property_capacities_data(id: int):
-	capacities = etl_rpc.region_properties_capacities(id)
-	return capacities
-
-@app.get("/region/{id}/capacities")
-def region_capacities_data(id: int):
+@app.get("/regions/{id}/capacities", response_model=RegionCapacities, tags=['region'])
+def region_capacities(id: int):
+	"""
+	Returs the capacities of a region, for every scraping. Set id to -1 to obtain data for all regions.
+	"""
 	capacities = etl_rc.region_capacities(id)
 	return capacities
 
-@app.get("/region/{id}/capacities/monthly/{scrapeDate}")
-def region_capacities_data(id: int, scrapeDate: str):
-	capacities = etl_rcm.region_capacities_monthly(id, scrapeDate)
+@app.get("/regions/{id}/capacities/monthly/{date}", response_model=RegionCapacitiesMonthly, tags=['region'])
+def region_capacities_monthly(id: int, date: datetime.date):
+	"""
+	Returns the capacities of a region for specified date by months. set id to -1 to obtain data for all regions.
+	"""
+	capacities = etl_rcm.region_capacities_monthly(id, date)
 	return capacities
 
-@app.get("/region/{id}/capacities/weekdays/{scrapeDate}")
-def region_capacities_data(id: int, scrapeDate: str):
-	capacities = etl_rcw.region_capacities_weekdays(id, scrapeDate)
+@app.get("/regions/{id}/capacities/daily/{date}", response_model=RegionCapacitiesDaily, tags=['region'])
+def region_capacities_daily(id: int, date: datetime.date):
+	"""
+	Returns the capacities of a region for specified date by days. set id to -1 to obtain data for all regions.
+	"""
+	capacities = etl_rcd.region_capacities_daily(id, date)
 	return capacities
 
-@app.get("/region/capacities/comparison/{id_1}/{id_2}")
-def region_capacities_data(id_1: int, id_2: int):
-	capacities = etl_rcc.region_capacities_comparison(id_1, id_2)
-	return capacities
-
-@app.get("/region/{id}/movingAverage/{startDate}")
-def region_capacities_data(id: int, startDate: str):
-	result = etl_rmA.region_movingAverage(id, startDate)
+@app.get("/regions/{id}/moving-average/{date}",  response_model=RegionMovingAverage, tags=['region'])
+def region_capacities_data(id: int, date: datetime.date):
+	"""
+	Returns the moving average of a region for specified date. set id to -1 to obtain data for all regions.
+	"""
+	result = etl_rmA.region_movingAverage(id, date)
 	return result
 
-@app.get("/region/{id}/base")
-def region_base_data(id: int):
-	return d.region_base_data(id).pl().to_dicts()
+@app.get("/regions/{id}/properties/capacities", response_model=RegionPropertiesCapacities, tags=['region'])
+def region_property_capacities(id: int):
+	"""
+	Returns the capacities of properties in region, for every scraping. set id to -1 to obtain data for all regions.
+	"""
+	capacities = etl_rpc.region_properties_capacities(id)
+	return capacities
+
+@app.get("/properties/growth", response_model=PropertiesGrowth, tags=['property'])
+def properties_growth():
+	"""
+	Returns the growth rate of found properties
+	"""
+	options = {"dates" : d.properties_growth().pl()['date'].to_list(), "total_all" : d.properties_growth().pl()['total_all'].to_list(),  "total_heidiland" : d.properties_growth().pl()['total_heidiland'].to_list(), "total_engadin" : d.properties_growth().pl()['total_engadin'].to_list(), "total_davos" : d.properties_growth().pl()['total_davos'].to_list(), "total_stmoritz" : d.properties_growth().pl()['total_stmoritz'].to_list()}
+	return options
+
+@app.get("/properties/geo", response_model=PropertiesGeo, tags=['property'])
+def properties_geo():
+	"""
+	Returns the geocoordinates of properties
+	"""
+	return {"properties": d.properties_geo().pl().to_dicts()}
+
+@app.get("/properties/{id}/base", response_model=PropertyBase, tags=['property'])
+def property_base_data(id: int):
+	"""
+	Returns basic information about a property.
+	"""
+	base = d.property_base_data(id).pl().to_dicts()
+	return {
+		"property_platform_id": base[0]['property_platform_id'], 
+		"first_found": str(base[0]['first_found']), 
+		"last_found": str(base[0]['last_found']), 
+		"latlng": base[0]['latlng'], 
+		"region_id": base[0]['region_id'], 
+		"region_name": base[0]['region_name']}
+
+@app.get("/properties/{id}/neighbours", response_model=PropertyNeighbours, tags=['property'])
+def property_neighbours(id: int):
+	"""
+	Returns the 10 nearest properties from given property.
+	"""
+	return {"neighbours" : etl_pn.property_neighbours(id)}
+
+@app.get("/properties/{id}/extractions", response_model=PropertyExtractions, tags=['property'])
+def property_extractions(id: int):
+	"""
+	Returns extracted data from given property.
+	"""
+	return {"extractions" : d.extractions_for(property_id = id).pl().cast({"date": pl.String}).to_dicts()}
+
+@app.get("/properties/{id}/capacities", response_model=PropertyCapacities, tags=['property'])
+def property_capacities_data(id: int):
+	"""
+	Returns capacities for given property.
+	"""
+	capacities = etl_pc.property_capacities(id)
+	return capacities
+
+@app.get("/properties/{id}/capacities/monthly/{date}", response_model=PropertyCapacitiesMonthly, tags=['property'])
+def property_capacities_data_monthly(id: int, date: datetime.date):
+	"""
+	Returns capacities for given property and date by month.
+	"""
+	capacities = etl_pcm.property_capacities_monthly(id, date)
+	return capacities
+
+@app.get("/properties/{id}/capacities/daily/{date}", response_model=PropertyCapacitiesDaily, tags=['property'])
+def property_capacities_data_daily(id: int, date: datetime.date):
+	"""
+	Returns capacities for given property and date by day.
+	"""
+	capacities = etl_pcd.property_capacities_daily(id, date)
+	return capacities
+
+
+
 

etl/src/data/database.py

@@ -298,7 +298,8 @@ class Database:
 			FROM 
 				consultancy_d.extractions
 			WHERE
-				type == 'calendar'
+				type == 'calendar' AND
+				calendar NOT NULL
 			ORDER BY
 				property_id
 		""")
@@ -319,13 +320,15 @@ class Database:
 				consultancy_d.seeds ON seeds.id = properties.seed_id 
 			LEFT JOIN 
 				consultancy_d.regions ON regions.id = seeds.region_id 
+			WHERE
+				calendar NOT NULL
 			""")
 
 	def extractions_for(self, property_id):
 		return self.connection.sql(f"""
 			SELECT 
 				JSON_EXTRACT(body, '$.content.days') as calendar,
-				created_at 
+				created_at as date
 			FROM 
 				consultancy_d.extractions
 			WHERE
@@ -402,7 +405,7 @@ class Database:
 				p.property_platform_id,
 				p.created_at as first_found,
 				p.last_found,
-				p.check_data,
+				p.check_data as latlng,
 				r.id as region_id,
 				r.name as region_name 
 			FROM
@@ -420,8 +423,8 @@ class Database:
 			where = f"WHERE r.id = {id}"
 		return self.connection.sql(f"""
 			SELECT
-				r.id as region_id,
-				r.name as region_name
+				r.id as id,
+				r.name as name
 			FROM
 				consultancy_d.regions r
 			{where}
@@ -463,7 +466,8 @@ class Database:
 				consultancy_d.properties ON properties.id = extractions.property_id 
 			WHERE
 				type == 'calendar' AND
-				properties.seed_id = {region_id}		
+				properties.seed_id = {region_id} AND
+				calendarBody NOT NULL		
 			""")
 
 	def singleScrape_of_region(self, region_id: int, scrape_date_min: str, scrape_date_max: str):
@@ -478,7 +482,8 @@ class Database:
 				type == 'calendar' AND
 				properties.seed_id = {region_id} AND 
 				extractions.created_at >= '{scrape_date_min}' AND
-				extractions.created_at < '{scrape_date_max}'	
+				extractions.created_at < '{scrape_date_max}' AND 
+				calendarBody NOT NULL		
 			""")
 
 	def singleScrape_of_global(self, scrape_date_min: str, scrape_date_max: str):
@@ -492,7 +497,8 @@ class Database:
 			WHERE
 				type == 'calendar' AND
 				extractions.created_at >= '{scrape_date_min}' AND
-				extractions.created_at < '{scrape_date_max}'	
+				extractions.created_at < '{scrape_date_max}' AND 
+				calendarBody NOT NULL		
 			""")
 
 	def singleScrape_of_region_scrapDate(self, region_id: int, scrape_date_min: str, scrape_date_max: str):
@@ -508,7 +514,8 @@ class Database:
 				type == 'calendar' AND
 				properties.seed_id = {region_id} AND 
 				extractions.created_at >= '{scrape_date_min}' AND
-				extractions.created_at < '{scrape_date_max}'	
+				extractions.created_at < '{scrape_date_max}' AND
+				calendarBody NOT NULL		
 			""")
 
 	def singleScrape_of_global_scrapDate(self, scrape_date_min: str, scrape_date_max: str):
@@ -523,7 +530,8 @@ class Database:
 			WHERE
 				type == 'calendar' AND
 				extractions.created_at >= '{scrape_date_min}' AND
-				extractions.created_at < '{scrape_date_max}'	
+				extractions.created_at < '{scrape_date_max}' AND
+				calendarBody NOT NULL	
 			""")
 
 	def capacity_global(self):
@@ -538,6 +546,8 @@ class Database:
 				consultancy_d.properties ON properties.id = extractions.property_id 
 			WHERE
 				type == 'calendar'
+			AND
+				calendarBody NOT NULL
 			""")
 
 	def capacity_comparison_of_region(self, region_id_1, region_id_2):
@@ -554,7 +564,8 @@ class Database:
 			WHERE
 				type == 'calendar' AND
 				(properties.seed_id = {region_id_1} OR 
-				properties.seed_id = {region_id_2})		
+				properties.seed_id = {region_id_2})	AND
+				calendarBody NOT NULL		
 			""")
 
 	def unique_scrapeDates(self):

etl/src/data/etl_pipelines.py

@@ -1,46 +0,0 @@
-import polars as pl
-import json
-from datetime import datetime, timedelta
-def expansion_Pipeline(df):
-    '''
-    Rearranges a given extractions Dataframe into an expanded Dataframe.
-    New Columns :propId, created_at calendar_date, calendar_value
-    :param df: Inputs from database.py/extractions or database.py/extractions_for functions
-    :return: expanded dataframe
-    '''
-    data = []
-
-    for row in df.iter_rows():
-        propId = row[1]
-        createdAt = row[2]
-        if row[0]:
-            temp = json.loads(row[0])
-            keys = temp.keys()
-            for key in keys:
-                out = [propId, createdAt.date(), datetime.strptime(key, '%Y-%m-%d').date(), temp[key]]
-                data.append(out)
-
-    df = pl.DataFrame(data, schema=["property_id", "created_at", "calendar_date", "calendar_value"])
-    return df
-
-def liveDates_Pipeline(df):
-    '''
-    Returns the expanded Dataframe with only the live data and no future data
-    :param df: Inputs from database.py/extractions or database.py/extractions_for functions
-    :return: expanded and filtered dataframe
-    '''
-    df = expansion_Pipeline(df)
-    print(df)
-    df = df.filter(pl.col("calendar_date") == pl.col("created_at")+timedelta(days=2))
-    return df
-
-def liveDates_PipelineFromExpanded(df):
-    '''
-    Filters an already expanded df and returns only the live data and no future data
-    NOTE: The actual live date and the next is always 0. The reason is most likely that it is forbidden to
-    book on the current or next day. Workaround: Compare with the day after tomorrow
-    :param df: Inputs from expansion_Pipeline
-    :return: expanded and filtered dataframe
-    '''
-    df = df.filter(pl.col("calendar_date") == pl.col("created_at")+timedelta(days=2))
-    return df

etl/src/data/etl_property_capacities.py

@@ -34,7 +34,7 @@ def property_capacities(id: int):
 		count_days.append(len(liste))
 
 	counts = pl.DataFrame({"count_days" : count_days, "sum" : sum_hor})
-	result = {"capacities": [], "dates": extractions['created_at'].cast(pl.Date).to_list() }
+	result = {"capacities": [], "dates": extractions['date'].cast(pl.Date).cast(pl.String).to_list() }
 	
 	for row in counts.rows(named=True):
 		max_capacity = row['count_days'] * 2

etl/src/data/etl_property_capacities_daily.py

@@ -7,7 +7,7 @@ from data import etl_cache
 
 d = data.load()
 
-def property_capacities_weekdays(id: int, scrapeDate: str):
+def property_capacities_daily(id: int, scrapeDate: str):
 	
 	file = f"etl_property_capacities_weekdays_{id}_{scrapeDate}.obj"
 	obj = etl_cache.openObj(file)
@@ -36,6 +36,6 @@ def property_capacities_weekdays(id: int, scrapeDate: str):
 	df_calendar = df_calendar.sort('weekday_num')
 	df_calendar = df_calendar.drop('weekday_num')
 
-	result = {"scraping-date": scrapeDate, "weekdays": df_calendar['weekday'].to_list(), 'capacities': df_calendar['column_0'].to_list()}
+	result = {"date": scrapeDate, "weekdays": df_calendar['weekday'].to_list(), 'capacities': df_calendar['column_0'].to_list()}
 	etl_cache.saveObj(file, result)
 	return result

etl/src/data/etl_property_capacities_monthly.py

@@ -24,12 +24,15 @@ def property_capacities_monthly(id: int, scrapeDate: str):
 		df_calendar = df_calendar.transpose()
 		df_calendar = df_calendar.with_columns(pl.Series(name="dates", values=columnTitles))
 		df_calendar = df_calendar.with_columns((pl.col("dates").str.to_date()))
+		df_calendar = df_calendar.with_columns((pl.col("dates").dt.month_end().dt.day().alias('numDays')))
 		df_calendar = df_calendar.with_columns((pl.col("dates").dt.strftime("%b") + " " + (pl.col("dates").dt.strftime("%Y"))).alias('date_short'))
 		df_calendar = df_calendar.with_columns((pl.col("dates").dt.strftime("%Y") + " " + (pl.col("dates").dt.strftime("%m"))).alias('dates'))
 
-	df_calendar = df_calendar.group_by(['dates', 'date_short']).agg(pl.col("column_0").sum())
+
+
+	df_calendar = df_calendar.group_by(['dates', 'date_short', 'numDays']).agg(pl.col("column_0").sum())
+	df_calendar = df_calendar.with_columns((pl.col("column_0") / pl.col("numDays") / 2 * 100).alias("column_0"))
 	df_calendar = df_calendar.sort('dates')
-	df_calendar = df_calendar.drop('dates')
-	result = {"scraping-date": scrapeDate, "months": df_calendar['date_short'].to_list(), 'capacities': df_calendar['column_0'].to_list()}
+	result = {"months": df_calendar['date_short'].to_list(), 'capacities': df_calendar['column_0'].to_list()}
 	etl_cache.saveObj(file, result)
 	return result

etl/src/data/etl_property_neighbours.py

@@ -19,11 +19,12 @@ def calcHaversinDistance(latMain, lonMain, lat, lon):
     dlat = latMain - lat
 
     a = sin(dlat / 2) ** 2 + cos(lat) * cos(latMain) * sin(dlon / 2) ** 2
-    c = 2 * asin(sqrt(a))  # 2 * atan2(sqrt(a), sqrt(1-a))
+    c = 2 * atan2(sqrt(a), sqrt(1-a))
     d = R * c
 
     return d
 
+
 def property_neighbours(id: int):
 
     file = f"etl_property_neighbours_{id}.obj"
@@ -46,7 +47,6 @@ def property_neighbours(id: int):
     extractions = extractions.filter(pl.col('id') != str(id))
 
     # Split coordinate into lat and lon
-    #extractions = extractions.with_columns((pl.col('coordinates').str.split(','))[0].alias("coordinates")).unnest("fields")
     extractions = extractions.with_columns(pl.col("coordinates").str.split_exact(",", 1).struct.rename_fields(["lat", "lon"]).alias("lat/lon")).unnest("lat/lon")
     extractions = extractions.drop('coordinates')
     extractions = extractions.with_columns(pl.col("lat").cast(pl.Float32))
@@ -67,7 +67,6 @@ def property_neighbours(id: int):
     extractions = extractions.sort("distances").head(10)
     extractions = extractions.drop('distances')
 
-    #result = {"ids": extractions['id'].to_list(), "lat": extractions['lat'].to_list(), "lon": extractions['lon'].to_list()}
     result = extractions.to_dicts()
     etl_cache.saveObj(file, result)
 

etl/src/data/etl_region_capacities_comparison.py

@@ -1,65 +0,0 @@
-import data
-import polars as pl
-from io import StringIO
-import numpy as np
-
-
-d = data.load()
-
-def region_capacities_comparison(id_1: int, id_2: int):
-    fulldf = d.capacity_comparison_of_region(id_1, id_2).pl()
-    # turn PropertyIDs and seedIDs to ints for sorting and filtering
-    fulldf = fulldf.cast({"property_id": int})
-    fulldf = fulldf.cast({"seed_id": int})
-    df_region1 = fulldf.filter(pl.col("seed_id") == id_1)
-    df_region2 = fulldf.filter(pl.col("seed_id") == id_2)
-    df_list = [df_region1, df_region2]
-    outDictList = []
-
-    for df in df_list:
-        # Get uniques for dates and propIDs and sort them
-        listOfDates = df.get_column("ScrapeDate").unique().sort()
-        listOfPropertyIDs = df.get_column("property_id").unique().sort()
-
-        # Create DFs from lists to merge later
-        datesDF = pl.DataFrame(listOfDates).with_row_index("date_index")
-        propIdDF = pl.DataFrame(listOfPropertyIDs).with_row_index("prop_index")
-
-        # Merge Dataframe to generate indices
-        df = df.join(datesDF, on='ScrapeDate')
-        df = df.join(propIdDF, on='property_id')
-        # Drop now useless columns ScrapeDate and property_id
-        df = df[['ScrapeDate', 'calendarBody', 'date_index', 'prop_index']]
-        # Calculate grid values
-        gridData = []
-        for row in df.rows(named=True):
-            # Return 0 for sum if calendar is null
-            if row['calendarBody']:
-                calDF = pl.read_json(StringIO(row['calendarBody']))
-                sum_hor = calDF.sum_horizontal()[0]
-            else:
-                sum_hor = 0
-            # With Index
-            # gridData.append([row['prop_index'], row['date_index'], sum_hor])
-            # With ScrapeDate
-            gridData.append([row['ScrapeDate'], row['date_index'], sum_hor])
-
-        gridData = np.array(gridData)
-        # get all values to calculate Max
-        allValues = gridData[:, 2].astype(int)
-        maxValue = np.max(allValues)
-        gridData[:, 2] = (allValues*100)/maxValue
-
-        # Return back to list
-        gridData = gridData.tolist()
-
-        # Cast listOfDates to datetime
-        listOfDates = listOfDates.cast(pl.Date).to_list()
-        listOfPropertyIDs = listOfPropertyIDs.to_list()
-
-        # Create JSON
-        tempDict = {'scrapeDates': listOfDates, 'property_ids': listOfPropertyIDs, 'values': gridData}
-        outDictList.append(tempDict)
-
-    outDict = {'region1': outDictList[0], 'region2': outDictList[1],}
-    return outDict

etl/src/data/etl_region_capacities_daily.py

@@ -8,16 +8,13 @@ from data import etl_cache
 
 d = data.load()
 
-def region_capacities_weekdays(id: int, scrapeDate_start: str):
+def region_capacities_daily(id: int, scrapeDate_start: str):
 
     file = f"etl_region_capacities_weekdays_{id}_{scrapeDate_start}.obj"
     obj = etl_cache.openObj(file)
     if obj:
         return obj
 
-    # String to Date
-    scrapeDate_start = datetime.strptime(scrapeDate_start, '%Y-%m-%d')
-
     # Get end date of start search-window
     scrapeDate_end = scrapeDate_start + timedelta(days=1)
 
@@ -62,6 +59,6 @@ def region_capacities_weekdays(id: int, scrapeDate_start: str):
     outDf = outDf.insert_column(1, means)
     outDf = outDf[['weekday', 'mean']]
 
-    result = {"scraping-date": scrapeDate, "weekdays": outDf['weekday'].to_list(),'capacities': outDf['mean'].to_list()}
+    result = {"weekdays": outDf['weekday'].to_list(),'capacities': outDf['mean'].to_list()}
     etl_cache.saveObj(file, result)
     return result

etl/src/data/etl_region_capacities_monthly.py

@@ -16,9 +16,6 @@ def region_capacities_monthly(id: int, scrapeDate_start: str):
     if obj:
         return obj
 
-    # String to Date
-    scrapeDate_start = datetime.strptime(scrapeDate_start, '%Y-%m-%d')
-
     # Get end date of start search-window
     scrapeDate_end = scrapeDate_start + timedelta(days=1)
 
@@ -41,11 +38,14 @@ def region_capacities_monthly(id: int, scrapeDate_start: str):
             df_calendar = df_calendar.transpose()
             df_calendar = df_calendar.with_columns(pl.Series(name="dates", values=columnTitles))
             df_calendar = df_calendar.with_columns((pl.col("dates").str.to_date()))
+            df_calendar = df_calendar.with_columns((pl.col("dates").dt.month_end().dt.day().alias('numDays')))
             df_calendar = df_calendar.with_columns((pl.col("dates").dt.strftime("%b") + " " + (pl.col("dates").dt.strftime("%Y"))).alias('date_short'))
             df_calendar = df_calendar.with_columns((pl.col("dates").dt.strftime("%Y") + " " + (pl.col("dates").dt.strftime("%m"))).alias('dates'))
-            df_calendar = df_calendar.group_by(['dates', 'date_short']).agg(pl.col("column_0").sum())
+            df_calendar = df_calendar.group_by(['dates', 'date_short','numDays']).agg(pl.col("column_0").sum())
+            df_calendar = df_calendar.with_columns((pl.col("column_0") / pl.col("numDays") / 2 * 100).alias("column_0"))
             df_calendar = df_calendar.sort('dates')
             df_calendar = df_calendar.drop('dates')
+            df_calendar = df_calendar.drop('numDays')
 
             df_calendar = df_calendar.rename({'column_0': str(counter)})
 
@@ -60,6 +60,6 @@ def region_capacities_monthly(id: int, scrapeDate_start: str):
     outDf = outDf.insert_column(1, means)
     outDf = outDf[['date_short', 'mean']]
 
-    result = {"scraping-date": scrapeDate, "months": outDf['date_short'].to_list(),'capacities': outDf['mean'].to_list()}
+    result = {"date": scrapeDate, "months": outDf['date_short'].to_list(),'capacities': outDf['mean'].to_list()}
     etl_cache.saveObj(file, result)
     return result

etl/src/data/etl_region_movAverage.py

@@ -8,7 +8,7 @@ from data import etl_cache
 
 d = data.load()
 
-def region_movingAverage(id: int, scrape_date_start_min: str):
+def region_movingAverage(id: int, scrape_date_start_min: datetime.date):
 
     file = f"etl_region_movingAverage_{id}_{scrape_date_start_min}.obj"
     obj = etl_cache.openObj(file)
@@ -31,9 +31,6 @@ def region_movingAverage(id: int, scrape_date_start_min: str):
     uniqueScrapeDates = uniqueScrapeDates.get_column('ScrapeDate').str.to_date()
     uniqueScrapeDates = uniqueScrapeDates.sort().to_list()
 
-    # String to Date
-    scrape_date_start_min = datetime.strptime(scrape_date_start_min, '%Y-%m-%d')
-
     # Get end date of start search-window
     scrape_date_start_max = scrape_date_start_min + timedelta(days=1)
 
@@ -41,7 +38,7 @@ def region_movingAverage(id: int, scrape_date_start_min: str):
     scrape_date_end_min = scrape_date_start_min + timedelta(days=timeOffset)
 
     # Get closest ScrapeDate
-    scrape_date_end_min = min(uniqueScrapeDates, key=lambda x: abs(x - scrape_date_end_min.date()))
+    scrape_date_end_min = min(uniqueScrapeDates, key=lambda x: abs(x - scrape_date_end_min))
     scrape_date_end_max = scrape_date_end_min + timedelta(days=1)
 
     final_end_date = scrape_date_end_min + timedelta(days=calcFrame)
@@ -78,7 +75,7 @@ def region_movingAverage(id: int, scrape_date_start_min: str):
                 calDF = calDF.with_columns((pl.col("dates").str.to_date()))
 
                 # Filter out all Data that's in the calculation frame
-                calDF = calDF.filter((pl.col("dates") >= scrape_date_end_min))
+                calDF = calDF.filter((pl.col("dates") >= (scrape_date_start_min + timedelta(days=1))))
                 calDF = calDF.filter((pl.col("dates") < final_end_date))
 
                 # Join all information into one Dataframe
@@ -93,6 +90,7 @@ def region_movingAverage(id: int, scrape_date_start_min: str):
         outDF = outDF.sort('dates')
         outDFList.append(outDF)
 
+
     # Calculate the horizontal Sum for all Dates
     arrayCunter = 0
     tempDFList = []
@@ -108,14 +106,13 @@ def region_movingAverage(id: int, scrape_date_start_min: str):
         tempDFList.append(newDF)
 
     # Join actual and predict Values
-    outDF = tempDFList[1].join(tempDFList[0], on='dates', how='outer')
+    outDF = tempDFList[0].join(tempDFList[1], on='dates', how='outer')
 
     # Rename Columns for clarity
-    outDF = outDF.drop_nulls()
     outDF = outDF.drop('dates_right')
 
     # sum_hor_predict is the data from the earlier ScrapeDate
-    outDF = outDF.rename({'sum_hor': 'sum_hor_actual', 'sum_hor_right': 'sum_hor_predict'})
+    outDF = outDF.rename({'sum_hor_right': 'sum_hor_actual', 'sum_hor': 'sum_hor_predict'})
 
     # Calculate Moving average from Start
     baseValues = outDF.get_column('sum_hor_predict').to_list()
@@ -133,6 +130,7 @@ def region_movingAverage(id: int, scrape_date_start_min: str):
 
     # Add moving_averages to df
     outDF = outDF.with_columns(moving_averages=pl.Series(moving_averages))
-    result = {'dates':outDF.get_column('dates').to_list(), 'cap_earlierTimeframe':outDF.get_column('sum_hor_predict').to_list(), 'cap_laterTimeframe':outDF.get_column('sum_hor_actual').to_list(), 'movAvg':outDF.get_column('moving_averages').to_list(),}
+
+    result = {'dates': outDF.get_column('dates').to_list(), 'capacities_timeframe_before': outDF.get_column('sum_hor_predict').to_list(), 'capacities_timeframe_after':outDF.get_column('sum_hor_actual').to_list(), 'capacities_moving_average':outDF.get_column('moving_averages').to_list(),}
     etl_cache.saveObj(file, result)
     return result

etl/src/data/etl_region_properties_capacities.py

@@ -52,13 +52,13 @@ def region_properties_capacities(id: int):
 
     for row in gridData.rows(named=True):
         capacity = (row['sum_hor']*100)/maxValue
-        values.append((row['scrape_date'], row['property_id'], capacity))
+        values.append({"date" : row['scrape_date'], "property_id": row['property_id'], "capacity": capacity})
 
     # Cast listOfDates to datetime
     listOfDates = listOfDates.cast(pl.Date).to_list()
     listOfPropertyIDs = listOfPropertyIDs.cast(pl.String).to_list()
 
     # Create JSON
-    outDict = {'scrapeDates': listOfDates, 'property_ids': listOfPropertyIDs, 'values': values}
+    outDict = {'dates': listOfDates, 'property_ids': listOfPropertyIDs, 'values': values}
     etl_cache.saveObj(file, outDict)
     return outDict

etl/src/mauro/Data_Analysis.py

@@ -1,199 +0,0 @@
-import MySQLdb
-import json
-from datetime import datetime, timedelta
-import numpy as np
-
-
-def getPropertyDataFromDB():
-    db = MySQLdb.connect(host="localhost",user="root",passwd="admin",db="consultancy")
-    cur = db.cursor()
-
-    cur.execute("SELECT id, seed_id, check_data "
-                "FROM properties ")
-    propData = cur.fetchall()
-    db.close()
-
-    return propData
-
-def getDataFromDB(propId):
-    '''
-    Function to get data from MySQL database filter with the given propId
-    :return: scrapeDates and calendarData
-    '''
-    db = MySQLdb.connect(host="localhost",user="root",passwd="admin",db="consultancy")
-    cur = db.cursor()
-
-    cur.execute("SELECT JSON_EXTRACT(header, '$.Date') "
-                "FROM extractions "
-                f"WHERE type='calendar' AND property_id = {propId};")
-    scrapeDates = cur.fetchall()
-
-    cur.execute("SELECT JSON_EXTRACT(body, '$.content.days') "
-                "FROM extractions "
-                f"WHERE type='calendar' AND property_id = {propId};")
-    calendarData = cur.fetchall()
-    db.close()
-
-    return scrapeDates, calendarData
-
-def getUniqueScrapeDates():
-    db = MySQLdb.connect(host="localhost",user="root",passwd="admin",db="consultancy")
-    cur = db.cursor()
-
-    cur.execute("SELECT JSON_EXTRACT(header, '$.Date') "
-                "FROM extractions "
-                f"WHERE type='calendar'")
-    uniqueScrapeDates = cur.fetchall()
-    db.close()
-
-    return uniqueScrapeDates
-
-def getPropsPerScrape(scrapeDate):
-    date = datetime.strptime(scrapeDate, '%Y-%m-%d')
-    end_date = date + timedelta(days=1)
-
-    db = MySQLdb.connect(host="localhost",user="root",passwd="admin",db="consultancy")
-    cur = db.cursor()
-
-    cur.execute("SELECT property_id "
-                "FROM extractions "
-                f"WHERE type='calendar' AND created_at > '{scrapeDate}' AND created_at < '{str(end_date)}'")
-    uniqueScrapeDates = cur.fetchall()
-    db.close()
-
-    return uniqueScrapeDates
-
-def getuniquePropIdFromDB():
-    '''
-    Function to get unique propId from MySQL database
-    :return: propList
-    '''
-    db = MySQLdb.connect(host="localhost",user="root",passwd="admin",db="consultancy")
-    cur = db.cursor()
-    cur.execute("SELECT DISTINCT property_id "
-                "FROM extractions;")
-    propIds = cur.fetchall()
-    db.close()
-
-    propList = []
-    for propId in propIds:
-        propList.append(propId[0])
-
-    return propList
-
-def reformatScrapeDates(scrapeDatesIn):
-    '''
-    Reformats the scrapeDates column to a shortened datetime format
-    :param scrapeDatesIn:
-    :return:
-    '''
-    scrapeDates = []
-    for row in scrapeDatesIn:
-        date = datetime.strptime(json.loads(row[0])[0], '%a, %d %b %Y %H:%M:%S %Z').date()
-        str = date.strftime('%Y-%m-%d')
-        scrapeDates.append(str)
-
-    return scrapeDates
-
-def checkForLostProprty(calendarData):
-    '''
-    Checks if there are "None" Entries in the calendarData meaning they were no longer found
-    :param calendarData:
-    :return: Boolean indicating if there are "None" Entries in the calendarData
-    '''
-    for row in calendarData:
-        if None in row:
-            return True
-    return False
-
-
-def getMinMaxDate(calendarData):
-    '''
-    Gets the min and max values from a calendar data
-    :param calendarData: get all calendar data from querry
-    :return: the minimal and maximal date
-    '''
-    #minimales und maximales Datum ermitteln
-    fullDateList = []
-    for row in calendarData:
-        tempJson = json.loads(row[0]).keys()
-        for key in tempJson:
-            #print(key)
-            fullDateList.append(datetime.strptime(key, '%Y-%m-%d').date())
-
-    end_dt = max(fullDateList)
-    start_dt = min(fullDateList)
-    delta = timedelta(days=1)
-    HeaderDates = []
-
-    while start_dt <= end_dt:
-        HeaderDates.append(start_dt)
-        start_dt += delta
-
-    return HeaderDates
-
-
-def creatDataMatrix(HeaderDates, calendarData):
-    '''
-    Creates the data matrix from a calendar data
-    :param HeaderDates: The list of all possible Dates in the dataset is used as the headers
-    :param calendarData: the main information from the sql querry
-    :return: data Matrix with all the dates in the dataset
-    '''
-    data = []
-    for row in calendarData:
-        tempList = [-1] * len(HeaderDates)
-        tempJson = json.loads(row[0])
-        for key in tempJson:
-            date = datetime.strptime(key, '%Y-%m-%d').date()
-            content = tempJson[key]
-            index = [i for i, x in enumerate(HeaderDates) if x == date]
-            tempList[index[0]] = content
-        data.append(tempList)
-
-    return data
-
-
-def getAccuracy(df, baseLine, compLine):
-    '''
-    Calculates the accuracy of a given dataframe with a given baseLine and compLine
-    :param df:
-    :param baseLine:
-    :param compLine:
-    :return: Accuracy: The percentage of dates that had the same information in both baseLine and compLine
-    '''
-    try:
-        df = df.iloc[[baseLine,compLine]]
-    except IndexError:
-        return -1
-    total = 0
-    noChange = 0
-    first = True
-    for series_name, series in df.items():
-        if first:
-            first = False
-        else:
-            total += 1
-            #print(series_name)
-            if series[baseLine] != -1:
-                if series[compLine] != -1:
-                    if series[baseLine] == series[compLine]:
-                        noChange += 1
-
-    accuracy = noChange / total
-    return accuracy
-
-def getMeanAccuracy(accList):
-    '''
-    Get the mean Accuracy of the entire timedelay of one property
-    :param accList: List of accuracy Values of a comparison
-    :return: Average of the accuracy values while ignoring the '-1' values
-    '''
-    out = []
-    for row in accList:
-        row = [x for x in row if x != -1]
-        out.append(np.average(row))
-    return out
-
-
-

etl/src/mauro/OLD_ConsultProj1/CalendarVisualization.py

@@ -1,83 +0,0 @@
-from datetime import datetime, timedelta
-import json
-
-import MySQLdb                                  #Version 2.2.4
-import pandas as pd                             #Version 2.2.3
-import plotly.express as px                     #Version 5.24.1
-
-db = MySQLdb.connect(host="localhost",user="root",passwd="admin",db="consultancy")
-cur = db.cursor()
-
-cur.execute("SELECT JSON_EXTRACT(header, '$.Date') "
-            "FROM extractions "
-            "WHERE type='calendar' AND property_id = 200;")
-dateoutput = cur.fetchall()
-
-
-cur.execute("SELECT JSON_EXTRACT(body, '$.content.days') "
-            "FROM extractions "
-            "WHERE type='calendar' AND property_id = 200;")
-
-output = cur.fetchall()
-db.close()
-
-#createScrapedate Liste
-ytickVals = list(range(0, 30, 5))
-scrapeDates = []
-#print(dateoutput)
-for row in dateoutput:
-    date = datetime.strptime(json.loads(row[0])[0], '%a, %d %b %Y %H:%M:%S %Z').date()
-    str = date.strftime('%d/%m/%Y')
-    scrapeDates.append(str)
-
-#minimales und maximales Datum ermitteln
-fullDateList = []
-for row in output:
-    tempJson = json.loads(row[0]).keys()
-    for key in tempJson:
-        #print(key)
-        fullDateList.append(datetime.strptime(key, '%Y-%m-%d').date())
-
-end_dt = max(fullDateList)
-start_dt = min(fullDateList)
-delta = timedelta(days=1)
-HeaderDates = []
-
-while start_dt <= end_dt:
-    HeaderDates.append(start_dt)
-    start_dt += delta
-
-#Create data-Matrix
-data = []
-for row in output:
-    tempList = [-1] * len(HeaderDates)
-    tempJson = json.loads(row[0])
-    for key in tempJson:
-        date = datetime.strptime(key, '%Y-%m-%d').date()
-        content = tempJson[key]
-        index = [i for i, x in enumerate(HeaderDates) if x == date]
-        tempList[index[0]] = content
-    data.append(tempList)
-
-#Transform to Dataframe for Plotly
-df = pd.DataFrame(data, columns=HeaderDates)
-
-#Generate Plotly Diagramm
-colScale = [[0, 'rgb(0, 0, 0)'], [0.33, 'rgb(204, 16, 16)'], [0.66, 'rgb(10, 102, 15)'], [1, 'rgb(17, 184, 26)']]
-fig = px.imshow(df, color_continuous_scale= colScale)
-lines = list(range(0,30,1))
-for i in lines:
-    #fig.add_hline(y=i+0.5, line_color="white")
-    fig.add_hline(y=i+0.5)
-
-fig.update_layout(yaxis = dict(tickfont = dict(size=50))),
-fig.update_layout(xaxis = dict(tickfont = dict(size=50)))
-fig.update_layout(xaxis_title="Verfügbarkeitsdaten Mietobjekt", yaxis_title="Scrapingvorgang")
-fig.update_xaxes(title_font_size=100, title_font_weight="bold")
-fig.update_yaxes(title_font_size=100, title_font_weight="bold")
-fig.update_layout(yaxis = dict(tickmode = 'array',tickvals = ytickVals, ticktext = scrapeDates))
-fig.update_xaxes(title_standoff = 80)
-fig.update_yaxes(title_standoff = 80)
-fig.update_layout(xaxis={'side': 'top'})
-fig.show()
-

etl/src/mauro/accuracyPerLocation.py

@@ -1,58 +0,0 @@
-import Data_Analysis as DA
-import pandas as pd
-
-accuracy = pd.read_csv(f'results/accMeanDf.csv')
-
-propData = DA.getPropertyDataFromDB()
-propData = pd.DataFrame(propData, columns =['property_id', 'region', 'geoLocation'])
-propData = propData.drop(columns=['geoLocation'])
-#print(propData)
-
-merge = pd.merge(propData, accuracy, on="property_id")
-#print(merge)
-
-#1 = Heidiland, 2 = Davos, 3 = Engadin 4 = St.Moritz
-
-heidiAcc = merge[merge['region'] == 1]
-davosAcc = merge[merge['region'] == 2]
-EngadAcc = merge[merge['region'] == 3]
-StMorAcc = merge[merge['region'] == 4]
-
-
-heidiMean = heidiAcc.mean(axis=0)
-davosMean = davosAcc.mean(axis=0)
-EngadMean = EngadAcc.mean(axis=0)
-StMorMean = StMorAcc.mean(axis=0)
-
-heidiSDev = heidiAcc.std(axis=0)
-davosSDev = davosAcc.std(axis=0)
-EngadSDev = EngadAcc.std(axis=0)
-StMorSDev = StMorAcc.std(axis=0)
-
-
-accuracyOverview = pd.DataFrame()
-
-accuracyOverview.insert(0, "St. Moritz StdDev", StMorSDev, True)
-accuracyOverview.insert(0, "St. Moritz Mean", StMorMean, True)
-accuracyOverview.insert(0, "Engadin StdDev", EngadSDev, True)
-accuracyOverview.insert(0, "Engadin Mean", EngadMean, True)
-accuracyOverview.insert(0, "Davos StdDev", davosSDev, True)
-accuracyOverview.insert(0, "Davos Mean", davosMean, True)
-accuracyOverview.insert(0, "Heidi StdDev", heidiSDev, True)
-accuracyOverview.insert(0, "Heidi Mean", heidiMean, True)
-
-
-accuracyOverview.drop(index=accuracyOverview.index[0], axis=0, inplace=True)
-accuracyOverview.drop(index=accuracyOverview.index[0], axis=0, inplace=True)
-accuracyOverview.to_csv('results/accuracyOverview.csv', index=True)
-
-#delete unused DF's
-del merge, accuracy, propData
-del heidiAcc, davosAcc, EngadAcc, StMorAcc
-del heidiMean, davosMean, EngadMean, StMorMean
-del heidiSDev, davosSDev, EngadSDev, StMorSDev
-
-
-print(accuracyOverview)
-
-

etl/src/mauro/createAccuracyValues.py

@@ -1,73 +0,0 @@
-import pandas as pd
-import os
-import re
-import numpy as np
-
-def getAccuracy(df, baseLine, compLine):
-    try:
-        df = df.iloc[[baseLine,compLine]]
-    except IndexError:
-        return -1
-    total = 0
-    noChange = 0
-    first = True
-    for series_name, series in df.items():
-        if first:
-            first = False
-        else:
-            total += 1
-            #print(series_name)
-            if series[baseLine] != -1:
-                if series[compLine] != -1:
-                    if series[baseLine] == series[compLine]:
-                        noChange += 1
-
-    accuracy = noChange / total
-    return accuracy
-
-def getMeanAccuracy(accList):
-    out = []
-    for row in accList:
-        row = [x for x in row if x != -1]
-        out.append(np.average(row))
-    return out
-
-deltaList = [1, 2, 10, 20]
-#1 = 1 Scrape Interval
-#2 = ca. 1 Woche
-#10 = 1 Monat (30Tage)
-#20 = 2 Monate
-
-
-directory = os.fsencode("dok")
-
-columnNames = ['property_id', 'timedelay_1', 'timedelay_2','timedelay_10','timedelay_20']
-accListDf = pd.DataFrame(columns = columnNames)
-accMeanDf = pd.DataFrame(columns = columnNames)
-
-
-for file in os.listdir(directory):
-    filename = os.fsdecode(file)
-    if filename.endswith(".csv"):
-        propId = re.findall("\d+", filename)[0]
-        print(propId)
-        df = pd.read_csv(f'dok/{filename}')
-        fullList = []
-        accList = []
-        #Loop though all deltas in the deltaList
-        for delta in deltaList:
-            accList = []
-            #Loop through all Dates as Baseline date
-            for i in range(df.shape[0]):
-                acc = getAccuracy(df, i, i+delta)
-                accList.append(acc)
-            fullList.append(accList)
-
-
-        meanList = getMeanAccuracy(fullList)
-        accListDf = accListDf._append({'property_id': propId, 'timedelay_1': fullList[0], 'timedelay_2': fullList[1], 'timedelay_10': fullList[2], 'timedelay_20': fullList[3]}, ignore_index=True)
-        accMeanDf = accMeanDf._append({'property_id': propId, 'timedelay_1': meanList[0], 'timedelay_2': meanList[1], 'timedelay_10': meanList[2], 'timedelay_20': meanList[3]}, ignore_index=True)
-
-
-accListDf.to_csv('results/accListDf.csv',  index=False)
-accMeanDf.to_csv('results/accMeanDf.csv',  index=False)

etl/src/mauro/createLostPropertyList.py

@@ -1,20 +0,0 @@
-import Data_Analysis as DA
-import csv
-
-propIds = DA.getuniquePropIdFromDB()
-
-lostProperties = []
-
-for propId in propIds:
-    print(propId)
-    scrapeDates, calendarData = DA.getDataFromDB(propId)
-    if DA.checkForLostProprty(calendarData):
-        lostProperties.append(propId)
-
-print(f"{len(lostProperties)} of {len(propIds)} properties are lost")
-
-with open('results/allLostProperties', 'w') as f:
-    write = csv.writer(f)
-    write.writerow(lostProperties)
-
-#Output: 221 of 1552 properties were lost at some point

etl/src/mauro/createPropCSV.py

@@ -1,28 +0,0 @@
-import Data_Analysis as DA
-import pandas as pd
-import os
-
-
-
-propIds = DA.getuniquePropIdFromDB()
-
-for propId in propIds:
-    name = f"dok/calendarData_prop{propId}.csv"
-    if not os.path.exists(name):
-        print(propId)
-        scrapeDates, calendarData = DA.getDataFromDB(propId)
-        if DA.checkForLostProprty(calendarData):
-            print(f"Lost Proprty: {propId}")
-        else:
-            scrapeDates = DA.reformatScrapeDates(scrapeDates)
-            HeaderDates = DA.getMinMaxDate(calendarData)
-            data = DA.creatDataMatrix(HeaderDates, calendarData)
-
-            # Transform to Dataframe for Plotly
-            df = pd.DataFrame(data, columns=HeaderDates)
-            df.insert(0, "ScrapeDate", scrapeDates, True)
-
-            df = df.drop(index=0)  # Irregulärer Abstand in den Scraping Zeiten (nur 2 Tage)
-            df = df.drop(df.columns[[1, 2]], axis=1)
-            df.to_csv(name, index=False)
-

etl/src/mauro/createPropertiesPerScrape.py

@@ -1,32 +0,0 @@
-import Data_Analysis as DA
-import pandas as pd
-
-#Alle Scrape Dates auslesen, umformatieren und doppelte Löschen
-uniqueScrapeDates = DA.getUniqueScrapeDates()
-uniqueScrapeDates = DA.reformatScrapeDates(uniqueScrapeDates)
-uniqueScrapeDates= list(dict.fromkeys(uniqueScrapeDates))
-#print(uniqueScrapeDates)
-
-#Liste der Listen der properties pro Scrape Datum erstellen
-fullPropList = []
-for date in uniqueScrapeDates:
-    propList = []
-    strDate = date
-    properties = DA.getPropsPerScrape(strDate)
-    for prop in properties:
-        propList.append(prop[0])
-        propList = list(dict.fromkeys(propList))
-    fullPropList.append(propList)
-    #print(propList)
-print(fullPropList)
-
-#zu DF umwandeln, mit Property ID's in the Spaltennamen und One-Hot-Encoding
-all_property_ids = sorted(set([item for sublist in fullPropList for item in sublist]))
-print(all_property_ids)
-df = pd.DataFrame(0, index=range(len(fullPropList)), columns=all_property_ids)
-for i, property_list in enumerate(fullPropList):
-    df.loc[i, property_list] = 1
-
-df.to_csv('results/PropertiesPerScrape.csv', index=True)
-
-print(df)

etl/src/mauro/data_quality.py

@@ -1,121 +0,0 @@
-from etl.src import data
-import json
-import polars as pl
-from datetime import datetime
-import matplotlib.pyplot as plt
-import numpy as np
-
-'''
-# Get Data from DB
-inst = data.load()
-
-df = inst.extractions_with_region().pl()
-print(df)
-
-counter = 0
-data = []
-for row in df.iter_rows():
-    property_id = row[1]
-    created_at = row[2].date()
-    dict = {'property_id': property_id, 'created_at': created_at, 'name': row[3]}
-
-    jsonStr = row[0]
-    if jsonStr:
-        calendarDict = json.loads(jsonStr)
-        for key in calendarDict:
-            dict[key] = calendarDict[key]
-
-    data.append(dict)
-
-dfNew = pl.from_dicts(data)
-dfNew.write_csv('results/data_quality.csv')
-print(dfNew)
-
-'''
-dfNew = pl.read_csv('results/data_quality.csv')
-dfNew = dfNew.with_columns(pl.col("created_at").map_elements(lambda x: datetime.strptime(x, "%Y-%m-%d").date()))
-
-# Create Row Means
-dfTemp = dfNew
-# Temporary Remove leading columns but save for later
-prop = dfTemp.get_column('property_id')
-dfTemp = dfTemp.drop('property_id')
-crea = dfTemp.get_column('created_at')
-dfTemp = dfTemp.drop('created_at')
-name = dfTemp.get_column('name')
-dfTemp = dfTemp.drop('name')
-dfTemp = dfTemp.with_columns(sum=pl.sum_horizontal(dfTemp.columns))
-sumCol = dfTemp.get_column('sum')
-
-# Create new DF with only property_id, created_at ,Location name and sum
-df = pl.DataFrame([prop, crea, name, sumCol])
-df = df.sort('created_at')
-
-# Create Full Copy
-# 0 = Alles
-# 1 = Heidiland
-# 2 = Davos
-# 3 = Engadin
-# 4 = St. Moritz
-filterList = ['Alle Regionen', 'Heidiland', 'Davos', 'Engadin', 'St. Moritz']
-
-filter = 4
-if filter != 0:
-    df = df.filter(pl.col("name") == filter)
-
-# Remove Location name
-df = df.drop('name')
-
-
-# Get unique property_ids
-propsIDs = df.unique(subset=["property_id"])
-propsIDs = propsIDs.get_column("property_id").to_list()
-propsIDs.sort()
-
-# create Matrix
-matrix = []
-for id in propsIDs:
-    dict = {}
-    temp = df.filter(pl.col("property_id") == id)
-    for row in temp.iter_rows():
-        dict[row[1].strftime('%Y-%m-%d')] = row[2]
-    matrix.append(dict)
-
-matrix = pl.DataFrame(matrix)
-dates = matrix.columns
-matrix = matrix.to_numpy()
-# normalized
-matrix = matrix/1111
-
-
-yRange = range(len(dates))
-xRange = range(len(propsIDs))
-matrix = matrix.T
-plt.imshow(matrix)
-plt.yticks(yRange[::5], dates[::5])
-plt.xticks(xRange[::10], propsIDs[::10])
-plt.title(filterList[filter])
-plt.xlabel("Property ID")
-plt.ylabel("Scrape Date")
-plt.colorbar()
-plt.tight_layout()
-
-# Create DiffMatrix
-diffMatrix = np.zeros((len(matrix)-1, len(matrix[0])))
-for y in range(len(matrix[0])):
-    for x in range(len(matrix)-1):
-        diffMatrix[x][y] = abs(matrix[x][y] - matrix[x+1][y])
-
-plt.figure()
-plt.imshow(diffMatrix, cmap="Reds")
-plt.yticks(yRange[::5], dates[::5])
-plt.xticks(xRange[::10], propsIDs[::10])
-plt.title(filterList[filter])
-plt.xlabel("Property ID")
-plt.ylabel("Scrape Date")
-plt.colorbar()
-plt.tight_layout()
-
-plt.show()
-
-