stoffelmauro/ConsultancyProject_2_ETL
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: stoffelmauro:5b97c7ead26dc220b0fa307e2e48bd680abe0c0f
Branches Tags
stoffelmauro:main
stoffelmauro:refactor-to-mono
..
compare: stoffelmauro:d598f9d861e205319fa9b93d7efd3dfc5f3b2a91
Branches Tags
stoffelmauro:main
stoffelmauro:refactor-to-mono

No commits in common. "5b97c7ead26dc220b0fa307e2e48bd680abe0c0f" and "d598f9d861e205319fa9b93d7efd3dfc5f3b2a91" have entirely different histories.
5b97c7ead2 ... d598f9d861

1 changed files with 7 additions and 16 deletions
Show Stats Expand all files Collapse all files

23
etl/src/data/database.py
View File

@ -298,8 +298,7 @@ class Database:
FROM FROM
consultancy_d.extractions consultancy_d.extractions
WHERE WHERE
type == 'calendar' AND type == 'calendar'
calendar NOT NULL
ORDER BY ORDER BY
property_id property_id
""") """)
@ -320,8 +319,6 @@ class Database:
consultancy_d.seeds ON seeds.id = properties.seed_id consultancy_d.seeds ON seeds.id = properties.seed_id
LEFT JOIN LEFT JOIN
consultancy_d.regions ON regions.id = seeds.region_id consultancy_d.regions ON regions.id = seeds.region_id
WHERE
calendar NOT NULL
""") """)
def extractions_for(self, property_id): def extractions_for(self, property_id):
@ -466,8 +463,7 @@ class Database:
consultancy_d.properties ON properties.id = extractions.property_id consultancy_d.properties ON properties.id = extractions.property_id
WHERE WHERE
type == 'calendar' AND type == 'calendar' AND
properties.seed_id = {region_id} AND properties.seed_id = {region_id}
calendarBody NOT NULL
""") """)
def singleScrape_of_region(self, region_id: int, scrape_date_min: str, scrape_date_max: str): def singleScrape_of_region(self, region_id: int, scrape_date_min: str, scrape_date_max: str):
@ -482,8 +478,7 @@ class Database:
type == 'calendar' AND type == 'calendar' AND
properties.seed_id = {region_id} AND properties.seed_id = {region_id} AND
extractions.created_at >= '{scrape_date_min}' AND extractions.created_at >= '{scrape_date_min}' AND
extractions.created_at < '{scrape_date_max}' AND extractions.created_at < '{scrape_date_max}'
calendarBody NOT NULL
""") """)
def singleScrape_of_global(self, scrape_date_min: str, scrape_date_max: str): def singleScrape_of_global(self, scrape_date_min: str, scrape_date_max: str):
@ -497,8 +492,7 @@ class Database:
WHERE WHERE
type == 'calendar' AND type == 'calendar' AND
extractions.created_at >= '{scrape_date_min}' AND extractions.created_at >= '{scrape_date_min}' AND
extractions.created_at < '{scrape_date_max}' AND extractions.created_at < '{scrape_date_max}'
calendarBody NOT NULL
""") """)
def singleScrape_of_region_scrapDate(self, region_id: int, scrape_date_min: str, scrape_date_max: str): def singleScrape_of_region_scrapDate(self, region_id: int, scrape_date_min: str, scrape_date_max: str):
@ -514,8 +508,7 @@ class Database:
type == 'calendar' AND type == 'calendar' AND
properties.seed_id = {region_id} AND properties.seed_id = {region_id} AND
extractions.created_at >= '{scrape_date_min}' AND extractions.created_at >= '{scrape_date_min}' AND
extractions.created_at < '{scrape_date_max}' AND extractions.created_at < '{scrape_date_max}'
calendarBody NOT NULL
""") """)
def singleScrape_of_global_scrapDate(self, scrape_date_min: str, scrape_date_max: str): def singleScrape_of_global_scrapDate(self, scrape_date_min: str, scrape_date_max: str):
@ -530,8 +523,7 @@ class Database:
WHERE WHERE
type == 'calendar' AND type == 'calendar' AND
extractions.created_at >= '{scrape_date_min}' AND extractions.created_at >= '{scrape_date_min}' AND
extractions.created_at < '{scrape_date_max}' AND extractions.created_at < '{scrape_date_max}'
calendarBody NOT NULL
""") """)
def capacity_global(self): def capacity_global(self):
@ -564,8 +556,7 @@ class Database:
WHERE WHERE
type == 'calendar' AND type == 'calendar' AND
(properties.seed_id = {region_id_1} OR (properties.seed_id = {region_id_1} OR
properties.seed_id = {region_id_2}) AND properties.seed_id = {region_id_2})
calendarBody NOT NULL
""") """)
def unique_scrapeDates(self): def unique_scrapeDates(self):