stoffelmauro
/
ConsultancyProject_2_ETL
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

global for movingAverage implemented

main
mmaurostoffel 2025-01-15 21:27:53 +01:00
parent 7e3862a578
commit c3ab7d8e2f
2 changed files with 26 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
etl/src/data/database.py
View File

@ -481,6 +481,20 @@ class Database:
extractions.created_at < '{scrape_date_max}' extractions.created_at < '{scrape_date_max}'
""") """)
def singleScrape_of_global(self, scrape_date_min: str, scrape_date_max: str):
return self.connection.sql(f"""
SELECT
JSON_EXTRACT(body, '$.content.days') as calendarBody,
FROM
consultancy_d.extractions
LEFT JOIN
consultancy_d.properties ON properties.id = extractions.property_id
WHERE
type == 'calendar' AND
extractions.created_at >= '{scrape_date_min}' AND
extractions.created_at < '{scrape_date_max}'
""")
def singleScrape_of_region_scrapDate(self, region_id: int, scrape_date_min: str, scrape_date_max: str): def singleScrape_of_region_scrapDate(self, region_id: int, scrape_date_min: str, scrape_date_max: str):
return self.connection.sql(f""" return self.connection.sql(f"""
SELECT SELECT

16
etl/src/data/etl_region_movAverage.py
View File

@ -46,11 +46,19 @@ def region_movingAverage(id: int, scrape_date_start_min: str):
final_end_date = scrape_date_end_min + timedelta(days=calcFrame) final_end_date = scrape_date_end_min + timedelta(days=calcFrame)
ex_start = d.singleScrape_of_region(id, scrape_date_start_min, scrape_date_start_max) # Get Data
ex_start_count = ex_start.shape[0] if id == -1:
ex_start = d.singleScrape_of_global(scrape_date_start_min, scrape_date_start_max)
ex_start_count = ex_start.shape[0]
ex_end = d.singleScrape_of_region(id, scrape_date_end_min, scrape_date_end_max) ex_end = d.singleScrape_of_global(scrape_date_end_min, scrape_date_end_max)
ex_end_count = ex_end.shape[0] ex_end_count = ex_end.shape[0]
else:
ex_start = d.singleScrape_of_region(id, scrape_date_start_min, scrape_date_start_max)
ex_start_count = ex_start.shape[0]
ex_end = d.singleScrape_of_region(id, scrape_date_end_min, scrape_date_end_max)
ex_end_count = ex_end.shape[0]
num_properties = [ex_start_count, ex_end_count] num_properties = [ex_start_count, ex_end_count]
start_end = [ex_start, ex_end] start_end = [ex_start, ex_end]