status(); $exception['headers'] = $response->headers(); $exception['body'] = $response->body(); $exceptionJSON = json_encode($exception); Exception::create([ 'exception' => $exceptionJSON, 'type' => $type, 'entity_id' => $entityId ]); } public static function getAllSeeds() { // get all seeds from model in random order. return Seed::select('id','uri')->inRandomOrder()->get(); } public static function getAllProperties() { // get all properties from model in random order. return Property::select('id','property_platform_id')->inRandomOrder()->get(); } public static function dispatchPropertyJobs() { $seeds = self::getAllSeeds(); foreach($seeds as $seed){ ScrapeProperty::dispatch($seed); } } public static function dispatchPropertyDataJobs() { $properties = self::getAllProperties(); foreach($properties as $property){ ScrapePropertyData::dispatch($property); } } public static function scrapeProperty($seed) { $response = Http::get($seed->uri); if($response->successful()){ $json = $response->json(); foreach($json['offers'] as $offer){ $property = Property::firstWhere('property_platform_id', $offer['id']); $geoLocation = implode(',', $offer['geoLocation']); if($property){ $property->last_found = now(); $property->save(); // check if geoLocation is the same as last crawl if($property->check_data !== $geoLocation){ Exception::create([ 'exception' => 'geoLocation was different: '.$geoLocation, 'entity_type' => 'property', 'entity_id' => $property->id ]); } }else{ Property::create([ 'property_platform_id' => $offer['id'], 'seed_id' => $seed->id, 'check_data' => $geoLocation, 'last_found' => now() ]); } } }else{ self::saveHttpException($response,'property', $seed->id); } } public static function scrapePropertyData($property){ // scrape offer details such as name etc. $offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id); if($offer->successful()){ Extraction::create([ 'property_id' => $property->id, 'type' => 'offer', 'body' => $offer->body(), 'header' => json_encode($offer->headers()) ]); }else{ self::saveHttpException($offer,'offer',$property->id); } // scrape price of property $price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id); if($price->successful()){ Extraction::create([ 'property_id' => $property->id, 'type' => 'price', 'body' => $price->body(), 'header' => json_encode($price->headers()) ]); }else{ self::saveHttpException($price,'price',$property->id); } // scrape calendar which contains occupancies $calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [ 'year' => date("Y"), 'month' => date("m") ]); if($calendar->successful()){ Extraction::create([ 'property_id' => $property->id, 'type' => 'calendar', 'body' => $calendar->body(), 'header' => json_encode($calendar->headers()) ]); }else{ self::saveHttpException($calendar,'price',$property->id); } } }