Cleanup
Giò
parent
fcfff433e2
commit
9cc2aad8dd
|
|
@ -1,62 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Filament\Resources;
|
|
||||||
|
|
||||||
use App\Filament\Resources\ExceptionsResource\Pages;
|
|
||||||
use App\Filament\Resources\ExceptionsResource\RelationManagers;
|
|
||||||
use App\Models\Exception;
|
|
||||||
use Filament\Forms;
|
|
||||||
use Filament\Forms\Form;
|
|
||||||
use Filament\Resources\Resource;
|
|
||||||
use Filament\Tables;
|
|
||||||
use Filament\Tables\Table;
|
|
||||||
use Illuminate\Database\Eloquent\Builder;
|
|
||||||
use Illuminate\Database\Eloquent\SoftDeletingScope;
|
|
||||||
|
|
||||||
class ExceptionsResource extends Resource
|
|
||||||
{
|
|
||||||
protected static ?string $model = Exception::class;
|
|
||||||
|
|
||||||
protected static ?string $navigationIcon = 'heroicon-o-rectangle-stack';
|
|
||||||
|
|
||||||
public static function form(Form $form): Form
|
|
||||||
{
|
|
||||||
return $form
|
|
||||||
->schema([
|
|
||||||
//
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function table(Table $table): Table
|
|
||||||
{
|
|
||||||
return $table
|
|
||||||
->columns([
|
|
||||||
//
|
|
||||||
])
|
|
||||||
->filters([
|
|
||||||
//
|
|
||||||
])
|
|
||||||
->actions([
|
|
||||||
Tables\Actions\EditAction::make(),
|
|
||||||
])
|
|
||||||
->bulkActions([
|
|
||||||
Tables\Actions\BulkActionGroup::make([
|
|
||||||
Tables\Actions\DeleteBulkAction::make(),
|
|
||||||
]),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function getRelations(): array
|
|
||||||
{
|
|
||||||
return [
|
|
||||||
//
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function getPages(): array
|
|
||||||
{
|
|
||||||
return [
|
|
||||||
'index' => Pages\ListExceptions::route('/'),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Filament\Resources\ExceptionsResource\Pages;
|
|
||||||
|
|
||||||
use App\Filament\Resources\ExceptionsResource;
|
|
||||||
use Filament\Actions;
|
|
||||||
use Filament\Resources\Pages\ListRecords;
|
|
||||||
|
|
||||||
class ListExceptions extends ListRecords
|
|
||||||
{
|
|
||||||
protected static string $resource = ExceptionsResource::class;
|
|
||||||
|
|
||||||
protected function getHeaderActions(): array
|
|
||||||
{
|
|
||||||
return [
|
|
||||||
//Actions\CreateAction::make(),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Filament\Resources;
|
|
||||||
|
|
||||||
use App\Filament\Resources\PropertiesResource\Pages;
|
|
||||||
use App\Filament\Resources\PropertiesResource\RelationManagers;
|
|
||||||
use App\Models\Property;
|
|
||||||
use Filament\Forms;
|
|
||||||
use Filament\Forms\Form;
|
|
||||||
use Filament\Resources\Resource;
|
|
||||||
use Filament\Tables;
|
|
||||||
use Filament\Tables\Table;
|
|
||||||
use Illuminate\Database\Eloquent\Builder;
|
|
||||||
use Illuminate\Database\Eloquent\SoftDeletingScope;
|
|
||||||
use Filament\Infolists\Components\TextEntry;
|
|
||||||
use Filament\Infolists\Infolist;
|
|
||||||
|
|
||||||
class PropertiesResource extends Resource
|
|
||||||
{
|
|
||||||
protected static ?string $model = Property::class;
|
|
||||||
protected static ?string $navigationIcon = 'heroicon-o-rectangle-stack';
|
|
||||||
|
|
||||||
public static function form(Form $form): Form
|
|
||||||
{
|
|
||||||
return $form
|
|
||||||
->schema([
|
|
||||||
//
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function table(Table $table): Table
|
|
||||||
{
|
|
||||||
return $table
|
|
||||||
->columns([
|
|
||||||
Tables\Columns\TextColumn::make('property_platform_id')->label('Platform ID')->searchable(),
|
|
||||||
Tables\Columns\TextColumn::make('last_found')->label('Last found')->searchable(),
|
|
||||||
])
|
|
||||||
->filters([
|
|
||||||
//
|
|
||||||
])
|
|
||||||
->actions([
|
|
||||||
Tables\Actions\ViewAction::make(),
|
|
||||||
])
|
|
||||||
->bulkActions([
|
|
||||||
Tables\Actions\BulkActionGroup::make([
|
|
||||||
Tables\Actions\DeleteBulkAction::make(),
|
|
||||||
]),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function infolist(Infolist $infolist): Infolist
|
|
||||||
{
|
|
||||||
return $infolist
|
|
||||||
->schema([
|
|
||||||
TextEntry::make('property_platform_id'),
|
|
||||||
TextEntry::make('last_found')
|
|
||||||
->dateTime(),
|
|
||||||
])
|
|
||||||
->columns(1)
|
|
||||||
->inlineLabel();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function getRelations(): array
|
|
||||||
{
|
|
||||||
return [
|
|
||||||
//RelationManagers\ExtractionsRelationManager::class,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function getPages(): array
|
|
||||||
{
|
|
||||||
return [
|
|
||||||
'index' => Pages\ListProperties::route('/'),
|
|
||||||
//'create' => Pages\CreateProperties::route('/create'),
|
|
||||||
'view' => Pages\ViewProperties::route('/{record}'),
|
|
||||||
//'edit' => Pages\EditProperties::route('/{record}/edit'),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Filament\Resources\PropertiesResource\Pages;
|
|
||||||
|
|
||||||
use App\Filament\Resources\PropertiesResource;
|
|
||||||
use Filament\Actions;
|
|
||||||
use Filament\Resources\Pages\ListRecords;
|
|
||||||
|
|
||||||
class ListProperties extends ListRecords
|
|
||||||
{
|
|
||||||
protected static string $resource = PropertiesResource::class;
|
|
||||||
|
|
||||||
protected function getHeaderActions(): array
|
|
||||||
{
|
|
||||||
return [
|
|
||||||
Actions\CreateAction::make(),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
<?php
|
|
||||||
namespace App\Filament\Resources\PropertiesResource\Pages;
|
|
||||||
use App\Filament\Resources\PropertiesResource;
|
|
||||||
use Filament\Infolists;
|
|
||||||
use Filament\Infolists\Infolist;
|
|
||||||
use Filament\Infolists\Components\TextEntry;
|
|
||||||
use Filament\Resources\Pages\ViewRecord;
|
|
||||||
|
|
||||||
class ViewProperties extends ViewRecord
|
|
||||||
{
|
|
||||||
protected static string $resource = PropertiesResource::class;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,55 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Filament\Resources\PropertiesResource\RelationManagers;
|
|
||||||
|
|
||||||
use Filament\Forms;
|
|
||||||
use Filament\Forms\Form;
|
|
||||||
use Filament\Resources\RelationManagers\RelationManager;
|
|
||||||
use Filament\Tables;
|
|
||||||
use Filament\Tables\Table;
|
|
||||||
use Illuminate\Database\Eloquent\Builder;
|
|
||||||
use Illuminate\Database\Eloquent\SoftDeletingScope;
|
|
||||||
|
|
||||||
class ExtractionsRelationManager extends RelationManager
|
|
||||||
{
|
|
||||||
protected static string $relationship = 'extractions';
|
|
||||||
|
|
||||||
public function isReadOnly(): bool
|
|
||||||
{
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function form(Form $form): Form
|
|
||||||
{
|
|
||||||
return $form
|
|
||||||
->schema([
|
|
||||||
Forms\Components\TextInput::make('property_platform_id')
|
|
||||||
->required()
|
|
||||||
->maxLength(255),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function table(Table $table): Table
|
|
||||||
{
|
|
||||||
return $table
|
|
||||||
->recordTitleAttribute('property_platform_id')
|
|
||||||
->columns([
|
|
||||||
Tables\Columns\TextColumn::make('property_platform_id'),
|
|
||||||
])
|
|
||||||
->filters([
|
|
||||||
//
|
|
||||||
])
|
|
||||||
->headerActions([
|
|
||||||
Tables\Actions\CreateAction::make(),
|
|
||||||
])
|
|
||||||
->actions([
|
|
||||||
Tables\Actions\EditAction::make(),
|
|
||||||
Tables\Actions\DeleteAction::make(),
|
|
||||||
])
|
|
||||||
->bulkActions([
|
|
||||||
Tables\Actions\BulkActionGroup::make([
|
|
||||||
Tables\Actions\DeleteBulkAction::make(),
|
|
||||||
]),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -9,9 +9,20 @@ use App\Jobs\ScrapeProperty;
|
||||||
use App\Jobs\ScrapePropertyData;
|
use App\Jobs\ScrapePropertyData;
|
||||||
use Illuminate\Support\Facades\Http;
|
use Illuminate\Support\Facades\Http;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This Class contains methods for scraping offers from the
|
||||||
|
* website e-domizil.ch.
|
||||||
|
**/
|
||||||
class Edomizil{
|
class Edomizil{
|
||||||
|
|
||||||
public static function saveHttpException($response, $type, $entityId){
|
/**
|
||||||
|
* Save an exception.
|
||||||
|
* @param string $response The respsonse form an exception e.g. 404
|
||||||
|
* @param enum $type Is either 'offer', 'price', 'calendar' or 'property'
|
||||||
|
* @param integer $entityId Has to be the id of the corresponding entity.
|
||||||
|
**/
|
||||||
|
public static function saveHttpException($response, $type, $entityId)
|
||||||
|
{
|
||||||
|
|
||||||
$exception = [];
|
$exception = [];
|
||||||
|
|
||||||
|
|
@ -29,52 +40,71 @@ class Edomizil{
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get seed urls.
|
||||||
|
* Get all seed urls (seeds.uris) in random order.
|
||||||
|
* @return Collection with seed urls.
|
||||||
|
**/
|
||||||
public static function getAllSeeds()
|
public static function getAllSeeds()
|
||||||
{
|
{
|
||||||
// get all seeds from model in random order.
|
|
||||||
return Seed::select('id','uri')->inRandomOrder()->get();
|
return Seed::select('id','uri')->inRandomOrder()->get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get property ids.
|
||||||
|
* Get all ids (properties.property_platform_id) in random order.
|
||||||
|
* @return Collection with property id
|
||||||
|
**/
|
||||||
public static function getAllProperties()
|
public static function getAllProperties()
|
||||||
{
|
{
|
||||||
// get all properties from model in random order.
|
// get all properties from model in random order.
|
||||||
return Property::select('id','property_platform_id')->inRandomOrder()->get();
|
return Property::select('id','property_platform_id')->inRandomOrder()->get();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function dispatchPropertyJobs()
|
/**
|
||||||
{
|
* Scrape for properties.
|
||||||
$seeds = self::getAllSeeds();
|
* Scrapes for properties form seed url and save them to the database.
|
||||||
foreach($seeds as $seed){
|
* @param $seed Seed
|
||||||
ScrapeProperty::dispatch($seed);
|
**/
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function dispatchPropertyDataJobs()
|
|
||||||
{
|
|
||||||
$properties = self::getAllProperties();
|
|
||||||
foreach($properties as $property){
|
|
||||||
ScrapePropertyData::dispatch($property);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function scrapeProperty($seed)
|
public static function scrapeProperty($seed)
|
||||||
{
|
{
|
||||||
|
|
||||||
$response = Http::get($seed->uri);
|
$response = Http::get($seed->uri);
|
||||||
|
|
||||||
if($response->successful()){
|
if($response->successful()){
|
||||||
|
|
||||||
$json = $response->json();
|
$json = $response->json();
|
||||||
|
|
||||||
|
/** Check if offers are findable in response */
|
||||||
|
if(!$json['offers']){
|
||||||
|
|
||||||
|
Exception::create([
|
||||||
|
'exception' => 'No offers found for'.$seed->uri,
|
||||||
|
'entity_type' => 'property',
|
||||||
|
'entity_id' => $property->id
|
||||||
|
]);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Iterate offers */
|
||||||
foreach($json['offers'] as $offer){
|
foreach($json['offers'] as $offer){
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if property with same id is already present in database.
|
||||||
|
* If already present check if the geoLocation was the same as the first time when found.
|
||||||
|
* Otherwise add property to database.
|
||||||
|
**/
|
||||||
$property = Property::firstWhere('property_platform_id', $offer['id']);
|
$property = Property::firstWhere('property_platform_id', $offer['id']);
|
||||||
$geoLocation = implode(',', $offer['geoLocation']);
|
$geoLocation = implode(',', $offer['geoLocation']);
|
||||||
|
|
||||||
if($property){
|
if($property){
|
||||||
|
|
||||||
|
/** Update last found attribute */
|
||||||
$property->last_found = now();
|
$property->last_found = now();
|
||||||
$property->save();
|
$property->save();
|
||||||
|
|
||||||
// check if geoLocation is the same as last crawl
|
/** check if geoLocation is the same as at creation time and save exception if not */
|
||||||
if($property->check_data !== $geoLocation){
|
if($property->check_data !== $geoLocation){
|
||||||
Exception::create([
|
Exception::create([
|
||||||
'exception' => 'geoLocation was different: '.$geoLocation,
|
'exception' => 'geoLocation was different: '.$geoLocation,
|
||||||
|
|
@ -82,6 +112,7 @@ class Edomizil{
|
||||||
'entity_id' => $property->id
|
'entity_id' => $property->id
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
}else{
|
}else{
|
||||||
Property::create([
|
Property::create([
|
||||||
'property_platform_id' => $offer['id'],
|
'property_platform_id' => $offer['id'],
|
||||||
|
|
@ -96,16 +127,22 @@ class Edomizil{
|
||||||
|
|
||||||
|
|
||||||
}else{
|
}else{
|
||||||
|
/** Save Exception if document could not be found */
|
||||||
self::saveHttpException($response,'property', $seed->id);
|
self::saveHttpException($response,'property', $seed->id);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract details from property.
|
||||||
|
* Scrapes for offer, price and calendar details from property and save the to extractions table (or exceptions when not found).
|
||||||
|
* @param $property Id of property (properties.property_platform_id)
|
||||||
|
**/
|
||||||
public static function scrapePropertyData($property){
|
public static function scrapePropertyData($property){
|
||||||
|
|
||||||
$result = [];
|
$result = [];
|
||||||
|
|
||||||
// scrape offer details such as name etc.
|
/** scrape offer details such as name, ammeneties, etc. */
|
||||||
$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id);
|
$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$property->property_platform_id);
|
||||||
|
|
||||||
if($offer->successful()){
|
if($offer->successful()){
|
||||||
|
|
@ -123,7 +160,7 @@ class Edomizil{
|
||||||
|
|
||||||
$result['offer'] = $offer->body();
|
$result['offer'] = $offer->body();
|
||||||
|
|
||||||
// scrape price of property
|
/** scrape for price details */
|
||||||
$price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id);
|
$price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$property->property_platform_id);
|
||||||
|
|
||||||
if($price->successful()){
|
if($price->successful()){
|
||||||
|
|
@ -141,7 +178,7 @@ class Edomizil{
|
||||||
|
|
||||||
$result['price'] = $price->body();
|
$result['price'] = $price->body();
|
||||||
|
|
||||||
// scrape calendar which contains occupancies
|
/** scrape for calendar details */
|
||||||
$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [
|
$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [
|
||||||
'year' => date("Y"),
|
'year' => date("Y"),
|
||||||
'month' => date("m")
|
'month' => date("m")
|
||||||
|
|
@ -166,5 +203,28 @@ class Edomizil{
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dispatch property jobs.
|
||||||
|
* Creates jobs for scraping new for properties
|
||||||
|
**/
|
||||||
|
public static function dispatchPropertyJobs()
|
||||||
|
{
|
||||||
|
$seeds = self::getAllSeeds();
|
||||||
|
foreach($seeds as $seed){
|
||||||
|
ScrapeProperty::dispatch($seed);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dispatch property data jobs.
|
||||||
|
* Creates jobs for scraping new for property detail data.
|
||||||
|
**/
|
||||||
|
public static function dispatchPropertyDataJobs()
|
||||||
|
{
|
||||||
|
$properties = self::getAllProperties();
|
||||||
|
foreach($properties as $property){
|
||||||
|
ScrapePropertyData::dispatch($property);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -1,66 +1 @@
|
||||||
{
|
|
||||||
"name": "laravel/laravel",
|
|
||||||
"type": "project",
|
|
||||||
"description": "The skeleton application for the Laravel framework.",
|
|
||||||
"keywords": ["laravel", "framework"],
|
|
||||||
"license": "MIT",
|
|
||||||
"require": {
|
|
||||||
"php": "^8.1",
|
|
||||||
"guzzlehttp/guzzle": "^7.2",
|
|
||||||
"laravel/framework": "^10.10",
|
|
||||||
"laravel/sanctum": "^3.3",
|
|
||||||
"laravel/tinker": "^2.8"
|
|
||||||
},
|
|
||||||
"require-dev": {
|
|
||||||
"fakerphp/faker": "^1.9.1",
|
|
||||||
"laravel/pint": "^1.0",
|
|
||||||
"laravel/sail": "^1.18",
|
|
||||||
"mockery/mockery": "^1.4.4",
|
|
||||||
"nunomaduro/collision": "^7.0",
|
|
||||||
"phpunit/phpunit": "^10.1",
|
|
||||||
"spatie/laravel-ignition": "^2.0"
|
|
||||||
},
|
|
||||||
"autoload": {
|
|
||||||
"psr-4": {
|
|
||||||
"App\\": "app/",
|
|
||||||
"Database\\Factories\\": "database/factories/",
|
|
||||||
"Database\\Seeders\\": "database/seeders/"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"autoload-dev": {
|
|
||||||
"psr-4": {
|
|
||||||
"Tests\\": "tests/"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"scripts": {
|
|
||||||
"post-autoload-dump": [
|
|
||||||
"Illuminate\\Foundation\\ComposerScripts::postAutoloadDump",
|
|
||||||
"@php artisan package:discover --ansi"
|
|
||||||
],
|
|
||||||
"post-update-cmd": [
|
|
||||||
"@php artisan vendor:publish --tag=laravel-assets --ansi --force"
|
|
||||||
],
|
|
||||||
"post-root-package-install": [
|
|
||||||
"@php -r \"file_exists('.env') || copy('.env.example', '.env');\""
|
|
||||||
],
|
|
||||||
"post-create-project-cmd": [
|
|
||||||
"@php artisan key:generate --ansi"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"extra": {
|
|
||||||
"laravel": {
|
|
||||||
"dont-discover": []
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"config": {
|
|
||||||
"optimize-autoloader": true,
|
|
||||||
"preferred-install": "dist",
|
|
||||||
"sort-packages": true,
|
|
||||||
"allow-plugins": {
|
|
||||||
"pestphp/pest-plugin": true,
|
|
||||||
"php-http/discovery": true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"minimum-stability": "stable",
|
|
||||||
"prefer-stable": true
|
|
||||||
}
|
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue