dianigionath
/
ConsultancyProject1_Auslastungsmodellierung
3
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleanup & beginning of custom commands.

main
Giò 2024-04-13 00:54:08 +02:00
parent f4a724618e
commit 83ecfa4593
6 changed files with 76 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
scraper/app/Console/Commands/scrapeProperty.php Normal file
View File

@ -0,0 +1,30 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
class scrapeProperty extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'scrape:property';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Scrapes for properties from seed';
/**
* Execute the console command.
*/
public function handle()
{
}
}

30
scraper/app/Console/Commands/scrapePropertyDate.php Normal file
View File

@ -0,0 +1,30 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
class scrapePropertyDate extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'scrape:propertydata';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Scrapes for property data from properties.';
/**
* Execute the console command.
*/
public function handle()
{
//
}
}

8
scraper/app/Jobs/ScrapeProperyData.php
View File

@ -12,12 +12,14 @@ class ScrapeProperyData implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $property;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct($property)
{
//
$this->property = $property;
}
/**
@ -25,6 +27,6 @@ class ScrapeProperyData implements ShouldQueue
*/
public function handle(): void
{
//
Edomizil::scrapePropertyData($this->property);
}
}

58
scraper/app/Jobs/scrapeProperties.php
View File

@ -1,58 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\Seed;
use App\Models\Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Http;
class scrapeProperties implements ShouldQueue, ShouldBeUnique
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $seed;
/**
* Create a new job instance.
*/
public function __construct($seed){
$this->seed = $seed;
}
/**
* Execute the job.
*/
public function handle(): void
{
// $response = Http::get($seed->uri);
$response = Http::get('https://diani.xyz/test.json');
$json = $response->json();
foreach($json['offers'] as $offer){
// Guessed ID to identify property on scraped platform
$property = Property::firstWhere('property_platform_id', $offer['id']);
// check if geoLocation hast the same values as the last time at crawltime
if($property && $property->check_data === implode(',', $offer['geoLocation'])){
$property->last_found = now();
$property->save();
}else if($property && $property->check_data !== implode(',', $offer['geoLocation'])){
dump('error');
}else{
Property::create([
'property_platform_id' => $offer['id'],
'seed_id' => $seed->id,
'check_data' => implode(',', $offer['geoLocation']),
'last_found' => now()
]);
}
}
}
}

22
scraper/app/Scraper/Edomizil.php
View File

@ -53,7 +53,7 @@ class Edomizil{
{
$properties = self::getAllProperties();
foreach($properties as $property){
ScrapePropertyData::dispatch($property->property_platform_id);
ScrapePropertyData::dispatch($property);
}
}
@ -96,42 +96,42 @@ class Edomizil{
}
}
public static function scrapePropertyData($propertyId){
public static function scrapePropertyData($property){
// scrape offer details such as name etc.
$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$propertyId);
$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id);
if($offer->successful()){
Extraction::create([
'property_id' => $propertyId,
'property_id' => $property->id,
'type' => 'offer',
'body' => $offer->body(),
'header' => json_encode($offer->headers())
]);
}else{
self::saveHttpException($offer,'offer',$propertyId);
self::saveHttpException($offer,'offer',$property->id);
}
// scrape price of property
$price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$propertyId);
$price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id);
if($price->successful()){
Extraction::create([
'property_id' => $propertyId,
'property_id' => $property->id,
'type' => 'price',
'body' => $price->body(),
'header' => json_encode($price->headers())
]);
}else{
self::saveHttpException($price,'price',$propertyId);
self::saveHttpException($price,'price',$property->id);
}
// scrape calendar which contains occupancies
$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$propertyId, [
$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [
'year' => date("Y"),
'month' => date("m")
]);
@ -139,14 +139,14 @@ class Edomizil{
if($calendar->successful()){
Extraction::create([
'property_id' => $propertyId,
'property_id' => $property->id,
'type' => 'calendar',
'body' => $calendar->body(),
'header' => json_encode($calendar->headers())
]);
}else{
self::saveHttpException($calendar,'price',$propertyId);
self::saveHttpException($calendar,'price',$property->id);
}
}

9
scraper/routes/web.php
View File

@ -7,12 +7,3 @@ Route::get('/', function () {
return view('index');
});
Route::get('/properties', function () {
Edomizil::dispatchPropertyJobs();
});
Route::get('/propertydata', function () {
Edomizil::scrapePropertyData(1);
//Edomizil::dispatchPropertyDataJobs();
});