dianigionath/ConsultancyProject1_Auslastungsmodellierung
3
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleanup

Browse Source
This commit is contained in:
Giò 2024-07-01 19:41:12 +02:00
parent fcfff433e2
commit 9cc2aad8dd
9 changed files with 88 additions and 8542 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
scraper/app/Filament/Resources/ExceptionsResource.php
View File

@ -1,62 +0,0 @@
<?php
namespace App\Filament\Resources;
use App\Filament\Resources\ExceptionsResource\Pages;
use App\Filament\Resources\ExceptionsResource\RelationManagers;
use App\Models\Exception;
use Filament\Forms;
use Filament\Forms\Form;
use Filament\Resources\Resource;
use Filament\Tables;
use Filament\Tables\Table;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\SoftDeletingScope;
class ExceptionsResource extends Resource
{
protected static ?string $model = Exception::class;
protected static ?string $navigationIcon = 'heroicon-o-rectangle-stack';
public static function form(Form $form): Form
{
return $form
->schema([
//
]);
}
public static function table(Table $table): Table
{
return $table
->columns([
//
])
->filters([
//
])
->actions([
Tables\Actions\EditAction::make(),
])
->bulkActions([
Tables\Actions\BulkActionGroup::make([
Tables\Actions\DeleteBulkAction::make(),
]),
]);
}
public static function getRelations(): array
{
return [
//
];
}
public static function getPages(): array
{
return [
'index' => Pages\ListExceptions::route('/'),
];
}
}

19
scraper/app/Filament/Resources/ExceptionsResource/Pages/ListExceptions.php
View File

@ -1,19 +0,0 @@
<?php
namespace App\Filament\Resources\ExceptionsResource\Pages;
use App\Filament\Resources\ExceptionsResource;
use Filament\Actions;
use Filament\Resources\Pages\ListRecords;
class ListExceptions extends ListRecords
{
protected static string $resource = ExceptionsResource::class;
protected function getHeaderActions(): array
{
return [
//Actions\CreateAction::make(),
];
}
}

79
scraper/app/Filament/Resources/PropertiesResource.php
View File

@ -1,79 +0,0 @@
<?php
namespace App\Filament\Resources;
use App\Filament\Resources\PropertiesResource\Pages;
use App\Filament\Resources\PropertiesResource\RelationManagers;
use App\Models\Property;
use Filament\Forms;
use Filament\Forms\Form;
use Filament\Resources\Resource;
use Filament\Tables;
use Filament\Tables\Table;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\SoftDeletingScope;
use Filament\Infolists\Components\TextEntry;
use Filament\Infolists\Infolist;
class PropertiesResource extends Resource
{
protected static ?string $model = Property::class;
protected static ?string $navigationIcon = 'heroicon-o-rectangle-stack';
public static function form(Form $form): Form
{
return $form
->schema([
//
]);
}
public static function table(Table $table): Table
{
return $table
->columns([
Tables\Columns\TextColumn::make('property_platform_id')->label('Platform ID')->searchable(),
Tables\Columns\TextColumn::make('last_found')->label('Last found')->searchable(),
])
->filters([
//
])
->actions([
Tables\Actions\ViewAction::make(),
])
->bulkActions([
Tables\Actions\BulkActionGroup::make([
Tables\Actions\DeleteBulkAction::make(),
]),
]);
}
public static function infolist(Infolist $infolist): Infolist
{
return $infolist
->schema([
TextEntry::make('property_platform_id'),
TextEntry::make('last_found')
->dateTime(),
])
->columns(1)
->inlineLabel();
}
public static function getRelations(): array
{
return [
//RelationManagers\ExtractionsRelationManager::class,
];
}
public static function getPages(): array
{
return [
'index' => Pages\ListProperties::route('/'),
//'create' => Pages\CreateProperties::route('/create'),
'view' => Pages\ViewProperties::route('/{record}'),
//'edit' => Pages\EditProperties::route('/{record}/edit'),
];
}
}

19
scraper/app/Filament/Resources/PropertiesResource/Pages/ListProperties.php
View File

@ -1,19 +0,0 @@
<?php
namespace App\Filament\Resources\PropertiesResource\Pages;
use App\Filament\Resources\PropertiesResource;
use Filament\Actions;
use Filament\Resources\Pages\ListRecords;
class ListProperties extends ListRecords
{
protected static string $resource = PropertiesResource::class;
protected function getHeaderActions(): array
{
return [
Actions\CreateAction::make(),
];
}
}

14
scraper/app/Filament/Resources/PropertiesResource/Pages/ViewProperties.php
View File

@ -1,14 +0,0 @@
<?php
namespace App\Filament\Resources\PropertiesResource\Pages;
use App\Filament\Resources\PropertiesResource;
use Filament\Infolists;
use Filament\Infolists\Infolist;
use Filament\Infolists\Components\TextEntry;
use Filament\Resources\Pages\ViewRecord;
class ViewProperties extends ViewRecord
{
protected static string $resource = PropertiesResource::class;
}

55
scraper/app/Filament/Resources/PropertiesResource/RelationManagers/ExtractionsRelationManager.php
View File

@ -1,55 +0,0 @@
<?php
namespace App\Filament\Resources\PropertiesResource\RelationManagers;
use Filament\Forms;
use Filament\Forms\Form;
use Filament\Resources\RelationManagers\RelationManager;
use Filament\Tables;
use Filament\Tables\Table;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\SoftDeletingScope;
class ExtractionsRelationManager extends RelationManager
{
protected static string $relationship = 'extractions';
public function isReadOnly(): bool
{
return true;
}
public function form(Form $form): Form
{
return $form
->schema([
Forms\Components\TextInput::make('property_platform_id')
->required()
->maxLength(255),
]);
}
public function table(Table $table): Table
{
return $table
->recordTitleAttribute('property_platform_id')
->columns([
Tables\Columns\TextColumn::make('property_platform_id'),
])
->filters([
//
])
->headerActions([
Tables\Actions\CreateAction::make(),
])
->actions([
Tables\Actions\EditAction::make(),
Tables\Actions\DeleteAction::make(),
])
->bulkActions([
Tables\Actions\BulkActionGroup::make([
Tables\Actions\DeleteBulkAction::make(),
]),
]);
}
}

104
scraper/app/Scraper/Edomizil.php
View File

@ -9,9 +9,20 @@ use App\Jobs\ScrapeProperty;
use App\Jobs\ScrapePropertyData; use App\Jobs\ScrapePropertyData;
use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Http;
/**
* This Class contains methods for scraping offers from the
* website e-domizil.ch.
**/
class Edomizil{ class Edomizil{
public static function saveHttpException($response, $type, $entityId){ /**
* Save an exception.
* @param string $response The respsonse form an exception e.g. 404
* @param enum $type Is either 'offer', 'price', 'calendar' or 'property'
* @param integer $entityId Has to be the id of the corresponding entity.
**/
public static function saveHttpException($response, $type, $entityId)
{
$exception = []; $exception = [];
@ -29,52 +40,71 @@ class Edomizil{
} }
/**
* Get seed urls.
* Get all seed urls (seeds.uris) in random order.
* @return Collection with seed urls.
**/
public static function getAllSeeds() public static function getAllSeeds()
{ {
// get all seeds from model in random order.
return Seed::select('id','uri')->inRandomOrder()->get(); return Seed::select('id','uri')->inRandomOrder()->get();
} }
/**
* Get property ids.
* Get all ids (properties.property_platform_id) in random order.
* @return Collection with property id
**/
public static function getAllProperties() public static function getAllProperties()
{ {
// get all properties from model in random order. // get all properties from model in random order.
return Property::select('id','property_platform_id')->inRandomOrder()->get(); return Property::select('id','property_platform_id')->inRandomOrder()->get();
} }
public static function dispatchPropertyJobs() /**
{ * Scrape for properties.
$seeds = self::getAllSeeds(); * Scrapes for properties form seed url and save them to the database.
foreach($seeds as $seed){ * @param $seed Seed
ScrapeProperty::dispatch($seed); **/
}
}
public static function dispatchPropertyDataJobs()
{
$properties = self::getAllProperties();
foreach($properties as $property){
ScrapePropertyData::dispatch($property);
}
}
public static function scrapeProperty($seed) public static function scrapeProperty($seed)
{ {
$response = Http::get($seed->uri); $response = Http::get($seed->uri);
if($response->successful()){ if($response->successful()){
$json = $response->json(); $json = $response->json();
/** Check if offers are findable in response */
if(!$json['offers']){
Exception::create([
'exception' => 'No offers found for'.$seed->uri,
'entity_type' => 'property',
'entity_id' => $property->id
]);
return;
}
/** Iterate offers */
foreach($json['offers'] as $offer){ foreach($json['offers'] as $offer){
/**
* Check if property with same id is already present in database.
* If already present check if the geoLocation was the same as the first time when found.
* Otherwise add property to database.
**/
$property = Property::firstWhere('property_platform_id', $offer['id']); $property = Property::firstWhere('property_platform_id', $offer['id']);
$geoLocation = implode(',', $offer['geoLocation']); $geoLocation = implode(',', $offer['geoLocation']);
if($property){ if($property){
/** Update last found attribute */
$property->last_found = now(); $property->last_found = now();
$property->save(); $property->save();
// check if geoLocation is the same as last crawl /** check if geoLocation is the same as at creation time and save exception if not */
if($property->check_data !== $geoLocation){ if($property->check_data !== $geoLocation){
Exception::create([ Exception::create([
'exception' => 'geoLocation was different: '.$geoLocation, 'exception' => 'geoLocation was different: '.$geoLocation,
@ -82,6 +112,7 @@ class Edomizil{
'entity_id' => $property->id 'entity_id' => $property->id
]); ]);
} }
}else{ }else{
Property::create([ Property::create([
'property_platform_id' => $offer['id'], 'property_platform_id' => $offer['id'],
@ -96,16 +127,22 @@ class Edomizil{
}else{ }else{
/** Save Exception if document could not be found */
self::saveHttpException($response,'property', $seed->id); self::saveHttpException($response,'property', $seed->id);
return 0; return 0;
} }
} }
/**
* Extract details from property.
* Scrapes for offer, price and calendar details from property and save the to extractions table (or exceptions when not found).
* @param $property Id of property (properties.property_platform_id)
**/
public static function scrapePropertyData($property){ public static function scrapePropertyData($property){
$result = []; $result = [];
// scrape offer details such as name etc. /** scrape offer details such as name, ammeneties, etc. */
$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id); $offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id);
if($offer->successful()){ if($offer->successful()){
@ -123,7 +160,7 @@ class Edomizil{
$result['offer'] = $offer->body(); $result['offer'] = $offer->body();
// scrape price of property /** scrape for price details */
$price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id); $price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id);
if($price->successful()){ if($price->successful()){
@ -141,7 +178,7 @@ class Edomizil{
$result['price'] = $price->body(); $result['price'] = $price->body();
// scrape calendar which contains occupancies /** scrape for calendar details */
$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [ $calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [
'year' => date("Y"), 'year' => date("Y"),
'month' => date("m") 'month' => date("m")
@ -166,5 +203,28 @@ class Edomizil{
} }
/**
* Dispatch property jobs.
* Creates jobs for scraping new for properties
**/
public static function dispatchPropertyJobs()
{
$seeds = self::getAllSeeds();
foreach($seeds as $seed){
ScrapeProperty::dispatch($seed);
}
} }
/**
* Dispatch property data jobs.
* Creates jobs for scraping new for property detail data.
**/
public static function dispatchPropertyDataJobs()
{
$properties = self::getAllProperties();
foreach($properties as $property){
ScrapePropertyData::dispatch($property);
}
}
}

67
scraper/composer.json
View File

@ -1,66 +1 @@
{
"name": "laravel/laravel",
"type": "project",
"description": "The skeleton application for the Laravel framework.",
"keywords": ["laravel", "framework"],
"license": "MIT",
"require": {
"php": "^8.1",
"guzzlehttp/guzzle": "^7.2",
"laravel/framework": "^10.10",
"laravel/sanctum": "^3.3",
"laravel/tinker": "^2.8"
},
"require-dev": {
"fakerphp/faker": "^1.9.1",
"laravel/pint": "^1.0",
"laravel/sail": "^1.18",
"mockery/mockery": "^1.4.4",
"nunomaduro/collision": "^7.0",
"phpunit/phpunit": "^10.1",
"spatie/laravel-ignition": "^2.0"
},
"autoload": {
"psr-4": {
"App\\": "app/",
"Database\\Factories\\": "database/factories/",
"Database\\Seeders\\": "database/seeders/"
}
},
"autoload-dev": {
"psr-4": {
"Tests\\": "tests/"
}
},
"scripts": {
"post-autoload-dump": [
"Illuminate\\Foundation\\ComposerScripts::postAutoloadDump",
"@php artisan package:discover --ansi"
],
"post-update-cmd": [
"@php artisan vendor:publish --tag=laravel-assets --ansi --force"
],
"post-root-package-install": [
"@php -r \"file_exists('.env') || copy('.env.example', '.env');\""
],
"post-create-project-cmd": [
"@php artisan key:generate --ansi"
]
},
"extra": {
"laravel": {
"dont-discover": []
}
},
"config": {
"optimize-autoloader": true,
"preferred-install": "dist",
"sort-packages": true,
"allow-plugins": {
"pestphp/pest-plugin": true,
"php-http/discovery": true
}
},
"minimum-stability": "stable",
"prefer-stable": true
}

8203
scraper/composer.lock generated
View File

File diff suppressed because it is too large Load Diff