173 lines
3.8 KiB
PHP
173 lines
3.8 KiB
PHP
<?php
|
|
namespace App\Scraper;
|
|
|
|
use App\Models\Seed;
|
|
use App\Models\Property;
|
|
use App\Models\Extraction;
|
|
use App\Models\Exception;
|
|
use App\Jobs\ScrapeProperty;
|
|
use App\Jobs\ScrapePropertyData;
|
|
use Illuminate\Support\Facades\Http;
|
|
|
|
class Edomizil{
|
|
|
|
public static function saveHttpException($response, $type, $entityId){
|
|
|
|
$exception = [];
|
|
|
|
$exception['status'] = $response->status();
|
|
$exception['headers'] = $response->headers();
|
|
$exception['body'] = $response->body();
|
|
|
|
$exceptionJSON = json_encode($exception);
|
|
|
|
Exception::create([
|
|
'exception' => $exceptionJSON,
|
|
'type' => $type,
|
|
'entity_id' => $entityId
|
|
]);
|
|
|
|
}
|
|
|
|
public static function getAllSeeds()
|
|
{
|
|
// get all seeds from model in random order.
|
|
return Seed::select('id','uri')->inRandomOrder()->get();
|
|
}
|
|
|
|
public static function getAllProperties()
|
|
{
|
|
// get all properties from model in random order.
|
|
return Property::select('id','property_platform_id')->inRandomOrder()->get();
|
|
}
|
|
|
|
public static function dispatchPropertyJobs()
|
|
{
|
|
$seeds = self::getAllSeeds();
|
|
foreach($seeds as $seed){
|
|
ScrapeProperty::dispatch($seed);
|
|
}
|
|
}
|
|
|
|
public static function dispatchPropertyDataJobs()
|
|
{
|
|
$properties = self::getAllProperties();
|
|
foreach($properties as $property){
|
|
ScrapePropertyData::dispatch($property);
|
|
}
|
|
}
|
|
|
|
public static function scrapeProperty($seed)
|
|
{
|
|
$response = Http::get($seed->uri);
|
|
|
|
if($response->successful()){
|
|
|
|
$json = $response->json();
|
|
|
|
foreach($json['offers'] as $offer){
|
|
|
|
$property = Property::firstWhere('property_platform_id', $offer['id']);
|
|
$geoLocation = implode(',', $offer['geoLocation']);
|
|
|
|
if($property){
|
|
$property->last_found = now();
|
|
$property->save();
|
|
|
|
// check if geoLocation is the same as last crawl
|
|
if($property->check_data !== $geoLocation){
|
|
Exception::create([
|
|
'exception' => 'geoLocation was different: '.$geoLocation,
|
|
'entity_type' => 'property',
|
|
'entity_id' => $property->id
|
|
]);
|
|
}
|
|
}else{
|
|
Property::create([
|
|
'property_platform_id' => $offer['id'],
|
|
'seed_id' => $seed->id,
|
|
'check_data' => $geoLocation,
|
|
'last_found' => now()
|
|
]);
|
|
}
|
|
}
|
|
|
|
return count($json['offers']);
|
|
|
|
|
|
}else{
|
|
self::saveHttpException($response,'property', $seed->id);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
public static function scrapePropertyData($property){
|
|
|
|
$result = [];
|
|
|
|
// scrape offer details such as name etc.
|
|
$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id);
|
|
|
|
if($offer->successful()){
|
|
|
|
Extraction::create([
|
|
'property_id' => $property->id,
|
|
'type' => 'offer',
|
|
'body' => $offer->body(),
|
|
'header' => json_encode($offer->headers())
|
|
]);
|
|
|
|
}else{
|
|
self::saveHttpException($offer,'offer',$property->id);
|
|
}
|
|
|
|
$result['offer'] = $offer->body();
|
|
|
|
// scrape price of property
|
|
$price = Http::withHeaders([
|
|
'Cookie' => 'c=CHF',
|
|
])->get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id);
|
|
|
|
if($price->successful()){
|
|
|
|
Extraction::create([
|
|
'property_id' => $property->id,
|
|
'type' => 'price',
|
|
'body' => $price->body(),
|
|
'header' => json_encode($price->headers())
|
|
]);
|
|
|
|
}else{
|
|
self::saveHttpException($price,'price',$property->id);
|
|
}
|
|
|
|
$result['price'] = $price->body();
|
|
|
|
// scrape calendar which contains occupancies
|
|
$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [
|
|
'year' => date("Y"),
|
|
'month' => date("m")
|
|
]);
|
|
|
|
if($calendar->successful()){
|
|
|
|
Extraction::create([
|
|
'property_id' => $property->id,
|
|
'type' => 'calendar',
|
|
'body' => $calendar->body(),
|
|
'header' => json_encode($calendar->headers())
|
|
]);
|
|
|
|
}else{
|
|
self::saveHttpException($calendar,'calendar',$property->id);
|
|
}
|
|
|
|
$result['calendar'] = $calendar->body();
|
|
|
|
return json_encode($result);
|
|
|
|
}
|
|
|
|
}
|
|
|