ConsultancyProject1_Auslastungsmodellierung/scraper/app/Scraper/Edomizil.php

<?php
namespace App\Scraper;

use App\Models\Seed;
use App\Models\Property;
use App\Models\Extraction;
use App\Models\Exception;
use App\Jobs\ScrapeProperty;
use App\Jobs\ScrapePropertyData;
use Illuminate\Support\Facades\Http;

class Edomizil{

	public static function saveHttpException($response, $type, $entityId){

		$exception = [];

		$exception['status'] = $response->status();
		$exception['headers'] = $response->headers();
		$exception['body'] = $response->body();

		$exceptionJSON = json_encode($exception);
		
		Exception::create([
			'exception' => $exceptionJSON,
			'type' => $type,
			'entity_id' => $entityId
		]);

	}

	public static function getAllSeeds()
	{
		// get all seeds from model in random order.
		return Seed::select('id','uri')->inRandomOrder()->get();
	}

	public static function getAllProperties()
	{
		// get all properties from model in random order.
		return Property::select('id','property_platform_id')->inRandomOrder()->get();
	}

	public static function dispatchPropertyJobs()
	{
		$seeds = self::getAllSeeds();
		foreach($seeds as $seed){
			ScrapeProperty::dispatch($seed);
		}
	}

	public static function dispatchPropertyDataJobs()
	{
		$properties = self::getAllProperties();
		foreach($properties as $property){
			ScrapePropertyData::dispatch($property);
		}
	}

	public static function scrapeProperty($seed)
	{
		$response = Http::get($seed->uri);
		
		if($response->successful()){

			$json = $response->json();

			foreach($json['offers'] as $offer){
				
				$property = Property::firstWhere('property_platform_id', $offer['id']);
				$geoLocation = implode(',', $offer['geoLocation']);

				if($property){
					$property->last_found = now();
					$property->save();

					// check if geoLocation is the same as last crawl
					if($property->check_data !== $geoLocation){
						Exception::create([
							'exception' => 'geoLocation was different: '.$geoLocation,
							'entity_type' => 'property',
							'entity_id' => $property->id
						]);
					}
				}else{
					Property::create([
						'property_platform_id' => $offer['id'],
						'seed_id' => $seed->id,
						'check_data' => $geoLocation,
						'last_found' => now()
					]);
				}
			}

			return count($json['offers']);


		}else{
			self::saveHttpException($response,'property', $seed->id);
			return 0;
		}
	}

	public static function scrapePropertyData($property){

		$result = [];

		// scrape offer details such as name etc.
		$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id);
		
		if($offer->successful()){		
			
			Extraction::create([
				'property_id' => $property->id,
				'type' => 'offer',
				'body' => $offer->body(),
				'header' => json_encode($offer->headers())
			]);		
		
		}else{
			self::saveHttpException($offer,'offer',$property->id);
		}

		$result['offer'] = $offer->body();

		// scrape price of property
		$price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id);

		if($price->successful()){

			Extraction::create([
				'property_id' => $property->id,
				'type' => 'price',
				'body' => $price->body(),
				'header' => json_encode($price->headers())
			]);

		}else{
			self::saveHttpException($price,'price',$property->id);
		}

		$result['price'] = $price->body();

		// scrape calendar which contains occupancies
		$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [
			'year' => date("Y"),
			'month' => date("m")
		]);
		
		if($calendar->successful()){

			Extraction::create([
				'property_id' => $property->id,
				'type' => 'calendar',
				'body' => $calendar->body(),
				'header' => json_encode($calendar->headers())
			]);

		}else{
			self::saveHttpException($calendar,'calendar',$property->id);
		}

		$result['calendar'] = $calendar->body();
		
		return json_encode($result);
	
	}

}