Semi functional Chaos.

main
Giò 2024-04-09 22:36:40 +02:00
parent c49327748d
commit 956ce5a95b
22 changed files with 657 additions and 228 deletions

7
docs/seeds.csv Normal file
View File

@ -0,0 +1,7 @@
URI,description
https://www.e-domizil.ch/search/632d3fb65adbe?_format=json&adults=1&duration=7,Heidiland
https://www.e-domizil.ch/search/5460aea91d044?_format=json&adults=1&duration=7,Davos (GR)
https://www.e-domizil.ch/search/5555b19e174fc?_format=json&adults=1&duration=7,Engadin
https://www.e-domizil.ch/search/5460aea9284b5?_format=json&adults=1&duration=7,St. Moritz
1 URI description
2 https://www.e-domizil.ch/search/632d3fb65adbe?_format=json&adults=1&duration=7 Heidiland
3 https://www.e-domizil.ch/search/5460aea91d044?_format=json&adults=1&duration=7 Davos (GR)
4 https://www.e-domizil.ch/search/5555b19e174fc?_format=json&adults=1&duration=7 Engadin
5 https://www.e-domizil.ch/search/5460aea9284b5?_format=json&adults=1&duration=7 St. Moritz

View File

@ -0,0 +1,33 @@
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Scraper\Edomizil;
class ScrapeProperty implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $uri;
/**
* Create a new job instance.
*/
public function __construct($uri)
{
$this->uri = $uri;
}
/**
* Execute the job.
*/
public function handle(): void
{
Edomizil::scrapeProperty($this->uri);
}
}

View File

@ -0,0 +1,30 @@
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ScrapeProperyData implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Create a new job instance.
*/
public function __construct()
{
//
}
/**
* Execute the job.
*/
public function handle(): void
{
//
}
}

View File

@ -0,0 +1,12 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
class Exception extends Model
{
use HasFactory;
protected $table = 'exceptions';
protected $fillable = ['exception', 'entity_type', 'entity_id'];
}

View File

@ -3,10 +3,12 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
class Seed extends Model
{
use HasFactory;
protected $table = 'seeds';
}

View File

@ -11,7 +11,10 @@ class AppServiceProvider extends ServiceProvider
*/
public function register(): void
{
//
if ($this->app->environment('local')) {
$this->app->register(\Laravel\Telescope\TelescopeServiceProvider::class);
$this->app->register(TelescopeServiceProvider::class);
}
}
/**

View File

@ -0,0 +1,64 @@
<?php
namespace App\Providers;
use Illuminate\Support\Facades\Gate;
use Laravel\Telescope\IncomingEntry;
use Laravel\Telescope\Telescope;
use Laravel\Telescope\TelescopeApplicationServiceProvider;
class TelescopeServiceProvider extends TelescopeApplicationServiceProvider
{
/**
* Register any application services.
*/
public function register(): void
{
// Telescope::night();
$this->hideSensitiveRequestDetails();
$isLocal = $this->app->environment('local');
Telescope::filter(function (IncomingEntry $entry) use ($isLocal) {
return $isLocal ||
$entry->isReportableException() ||
$entry->isFailedRequest() ||
$entry->isFailedJob() ||
$entry->isScheduledTask() ||
$entry->hasMonitoredTag();
});
}
/**
* Prevent sensitive request details from being logged by Telescope.
*/
protected function hideSensitiveRequestDetails(): void
{
if ($this->app->environment('local')) {
return;
}
Telescope::hideRequestParameters(['_token']);
Telescope::hideRequestHeaders([
'cookie',
'x-csrf-token',
'x-xsrf-token',
]);
}
/**
* Register the Telescope gate.
*
* This gate determines who can access Telescope in non-local environments.
*/
protected function gate(): void
{
Gate::define('viewTelescope', function ($user) {
return in_array($user->email, [
//
]);
});
}
}

View File

@ -4,71 +4,99 @@ namespace App\Scraper;
use App\Models\Seed;
use App\Models\Property;
use App\Models\Occupancy;
use App\Jobs\scrapeProperties;
use App\Models\Exception;
use App\Jobs\ScrapeProperty;
use App\Jobs\ScrapePropertyData;
use Illuminate\Support\Facades\Http;
class Edomizil{
public static function getProperties(){
public static function getAllSeeds()
{
// get all properties from model in random order.
return Seed::select('id','uri')->inRandomOrder()->get();
}
$seeds = Seed::select('id','uri')->inRandomOrder()->get();
foreach ($seeds as $seed) {
dispatch(new scrapeProperties($seed));
public static function getAllProperties()
{
// get all properties from model in random order.
return Property::select('id','property_platform_id')->inRandomOrder()->get();
}
public static function dispatchPropertyJobs()
{
$seeds = self::getAllSeeds();
foreach($seeds as $seed){
ScrapeProperty::dispatch($seed->uri);
}
/*
foreach($seeds::all() as $seed){
}
$response = Http::get($seed->uri);
$json = $response->json();
public static function dispatchPropertyDataJobs()
{
$properties = self::getAllProperties();
foreach($properties as $property){
dump($property->property_platform_id);
// ScrapePropertyData::dispatch($property->property_platform_id);
}
}
foreach($json['offers'] as $offer){
public static function scrapeProperty($uri)
{
//$response = Http::get($seed->uri);
$response = Http::get('https://diani.xyz/test_2.json');
$json = $response->json();
$property = Property::firstWhere('property_platform_id', $offer['id']);
foreach($json['offers'] as $offer){
$property = Property::firstWhere('property_platform_id', $offer['id']);
$geoLocation = implode(',', $offer['geoLocation']);
if($property && $property->check_data === implode(',', $offer['geoLocation'])){
$property->last_found = now();
$property->save();
}else if($property && $property->check_data !== implode(',', $offer['geoLocation'])){
dump('error');
}else{
Property::create([
'property_platform_id' => $offer['id'],
'seed_id' => $seed->id,
'check_data' => implode(',', $offer['geoLocation']),
'last_found' => now()
if($property){
$property->last_found = now();
$property->save();
if($property->check_data !== $geoLocation){
Exception::create([
'exception' => 'geoLocation was different: '.$geoLocation,
'entity_type' => 'property',
'entity_id' => $offer['id']
]);
}
}else{
Property::create([
'property_platform_id' => $offer['id'],
'seed_id' => $seed->id,
'check_data' => $geoLocation,
'last_found' => now()
]);
}
}
}
public static function scrapeOccupancy($propertyId){
/*
$calendar = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$propertyId, [
'year' => date("Y"),
'month' => date("m")
]);
$data_cal = $calendar->json();
$price = Http::get('https://www.e-domizil.ch/booking/checkout/priceDetails/'.$propertyId);
$data_price = $price->json();
$offer = Http::get('https://www.e-domizil.ch/rental/offer/'.$propertyId);
$data_offer = $offer->json();
*/
/*
$data = $response->json();
Occupancy::create([
'property_id' => $property->id,
'occupancy' => json_encode($data['content']['days']),
'header' => json_encode($response->headers())
]);
*/
}
public static function getOccupancies(){
// get all properties from model in random order.
$properties = Property::select('id','property_platform_id')->inRandomOrder()->get();
foreach($properties as $property){
dump($property->id);
/*
$response = Http::get('https://www.e-domizil.ch/api/v2/calendar/'.$property->property_platform_id, [
'year' => date("Y")
]);
$data = $response->json();
Occupancy::create([
'property_id' => $property->id,
'occupancy' => json_encode($data['content']['days']),
'header' => json_encode($response->headers())
]);
*/
}
}
}

View File

@ -1,6 +1,5 @@
<?php
return [
App\Providers\AppServiceProvider::class,
App\Providers\HorizonServiceProvider::class,
App\Providers\AppServiceProvider::class
];

View File

@ -13,6 +13,7 @@
"fakerphp/faker": "^1.23",
"laravel/pint": "^1.13",
"laravel/sail": "^1.26",
"laravel/telescope": "^5.0",
"mockery/mockery": "^1.6",
"nunomaduro/collision": "^8.0",
"phpunit/phpunit": "^10.5",
@ -49,7 +50,9 @@
},
"extra": {
"laravel": {
"dont-discover": []
"dont-discover": [
"laravel/telescope"
]
}
},
"config": {

70
scraper/composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "968f73bcf5520ee401d641d0d79555ec",
"content-hash": "ca35f12b69f5e2117186d9408e661df9",
"packages": [
{
"name": "brick/math",
@ -5999,6 +5999,74 @@
},
"time": "2024-03-20T20:09:31+00:00"
},
{
"name": "laravel/telescope",
"version": "v5.0.0",
"source": {
"type": "git",
"url": "https://github.com/laravel/telescope.git",
"reference": "e398fae4836e68b3d3734d7cdc37622eaf90fdd9"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/laravel/telescope/zipball/e398fae4836e68b3d3734d7cdc37622eaf90fdd9",
"reference": "e398fae4836e68b3d3734d7cdc37622eaf90fdd9",
"shasum": ""
},
"require": {
"ext-json": "*",
"laravel/framework": "^8.37|^9.0|^10.0|^11.0",
"php": "^8.0",
"symfony/var-dumper": "^5.0|^6.0|^7.0"
},
"require-dev": {
"ext-gd": "*",
"guzzlehttp/guzzle": "^6.0|^7.0",
"laravel/octane": "^1.4|^2.0|dev-develop",
"orchestra/testbench": "^6.40|^7.37|^8.17|^9.0",
"phpstan/phpstan": "^1.10",
"phpunit/phpunit": "^9.0|^10.5"
},
"type": "library",
"extra": {
"laravel": {
"providers": [
"Laravel\\Telescope\\TelescopeServiceProvider"
]
}
},
"autoload": {
"psr-4": {
"Laravel\\Telescope\\": "src/",
"Laravel\\Telescope\\Database\\Factories\\": "database/factories/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Taylor Otwell",
"email": "taylor@laravel.com"
},
{
"name": "Mohamed Said",
"email": "mohamed@laravel.com"
}
],
"description": "An elegant debug assistant for the Laravel framework.",
"keywords": [
"debugging",
"laravel",
"monitoring"
],
"support": {
"issues": "https://github.com/laravel/telescope/issues",
"source": "https://github.com/laravel/telescope/tree/v5.0.0"
},
"time": "2024-03-12T14:17:54+00:00"
},
{
"name": "mockery/mockery",
"version": "1.6.11",

View File

@ -0,0 +1,189 @@
<?php
use Laravel\Telescope\Http\Middleware\Authorize;
use Laravel\Telescope\Watchers;
return [
/*
|--------------------------------------------------------------------------
| Telescope Domain
|--------------------------------------------------------------------------
|
| This is the subdomain where Telescope will be accessible from. If the
| setting is null, Telescope will reside under the same domain as the
| application. Otherwise, this value will be used as the subdomain.
|
*/
'domain' => env('TELESCOPE_DOMAIN'),
/*
|--------------------------------------------------------------------------
| Telescope Path
|--------------------------------------------------------------------------
|
| This is the URI path where Telescope will be accessible from. Feel free
| to change this path to anything you like. Note that the URI will not
| affect the paths of its internal API that aren't exposed to users.
|
*/
'path' => env('TELESCOPE_PATH', 'telescope'),
/*
|--------------------------------------------------------------------------
| Telescope Storage Driver
|--------------------------------------------------------------------------
|
| This configuration options determines the storage driver that will
| be used to store Telescope's data. In addition, you may set any
| custom options as needed by the particular driver you choose.
|
*/
'driver' => env('TELESCOPE_DRIVER', 'database'),
'storage' => [
'database' => [
'connection' => env('DB_CONNECTION', 'mysql'),
'chunk' => 1000,
],
],
/*
|--------------------------------------------------------------------------
| Telescope Master Switch
|--------------------------------------------------------------------------
|
| This option may be used to disable all Telescope watchers regardless
| of their individual configuration, which simply provides a single
| and convenient way to enable or disable Telescope data storage.
|
*/
'enabled' => env('TELESCOPE_ENABLED', true),
/*
|--------------------------------------------------------------------------
| Telescope Route Middleware
|--------------------------------------------------------------------------
|
| These middleware will be assigned to every Telescope route, giving you
| the chance to add your own middleware to this list or change any of
| the existing middleware. Or, you can simply stick with this list.
|
*/
'middleware' => [
'web',
Authorize::class,
],
/*
|--------------------------------------------------------------------------
| Allowed / Ignored Paths & Commands
|--------------------------------------------------------------------------
|
| The following array lists the URI paths and Artisan commands that will
| not be watched by Telescope. In addition to this list, some Laravel
| commands, like migrations and queue commands, are always ignored.
|
*/
'only_paths' => [
// 'api/*'
],
'ignore_paths' => [
'livewire*',
'nova-api*',
'pulse*',
],
'ignore_commands' => [
//
],
/*
|--------------------------------------------------------------------------
| Telescope Watchers
|--------------------------------------------------------------------------
|
| The following array lists the "watchers" that will be registered with
| Telescope. The watchers gather the application's profile data when
| a request or task is executed. Feel free to customize this list.
|
*/
'watchers' => [
Watchers\BatchWatcher::class => env('TELESCOPE_BATCH_WATCHER', true),
Watchers\CacheWatcher::class => [
'enabled' => env('TELESCOPE_CACHE_WATCHER', true),
'hidden' => [],
],
Watchers\ClientRequestWatcher::class => env('TELESCOPE_CLIENT_REQUEST_WATCHER', true),
Watchers\CommandWatcher::class => [
'enabled' => env('TELESCOPE_COMMAND_WATCHER', true),
'ignore' => [],
],
Watchers\DumpWatcher::class => [
'enabled' => env('TELESCOPE_DUMP_WATCHER', true),
'always' => env('TELESCOPE_DUMP_WATCHER_ALWAYS', false),
],
Watchers\EventWatcher::class => [
'enabled' => env('TELESCOPE_EVENT_WATCHER', true),
'ignore' => [],
],
Watchers\ExceptionWatcher::class => env('TELESCOPE_EXCEPTION_WATCHER', true),
Watchers\GateWatcher::class => [
'enabled' => env('TELESCOPE_GATE_WATCHER', true),
'ignore_abilities' => [],
'ignore_packages' => true,
'ignore_paths' => [],
],
Watchers\JobWatcher::class => env('TELESCOPE_JOB_WATCHER', true),
Watchers\LogWatcher::class => [
'enabled' => env('TELESCOPE_LOG_WATCHER', true),
'level' => 'error',
],
Watchers\MailWatcher::class => env('TELESCOPE_MAIL_WATCHER', true),
Watchers\ModelWatcher::class => [
'enabled' => env('TELESCOPE_MODEL_WATCHER', true),
'events' => ['eloquent.*'],
'hydrations' => true,
],
Watchers\NotificationWatcher::class => env('TELESCOPE_NOTIFICATION_WATCHER', true),
Watchers\QueryWatcher::class => [
'enabled' => env('TELESCOPE_QUERY_WATCHER', true),
'ignore_packages' => true,
'ignore_paths' => [],
'slow' => 100,
],
Watchers\RedisWatcher::class => env('TELESCOPE_REDIS_WATCHER', true),
Watchers\RequestWatcher::class => [
'enabled' => env('TELESCOPE_REQUEST_WATCHER', true),
'size_limit' => env('TELESCOPE_RESPONSE_SIZE_LIMIT', 64),
'ignore_http_methods' => [],
'ignore_status_codes' => [],
],
Watchers\ScheduleWatcher::class => env('TELESCOPE_SCHEDULE_WATCHER', true),
Watchers\ViewWatcher::class => env('TELESCOPE_VIEW_WATCHER', true),
],
];

View File

@ -0,0 +1,70 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Get the migration connection name.
*/
public function getConnection(): string|null
{
return config('telescope.storage.database.connection');
}
/**
* Run the migrations.
*/
public function up(): void
{
$schema = Schema::connection($this->getConnection());
$schema->create('telescope_entries', function (Blueprint $table) {
$table->bigIncrements('sequence');
$table->uuid('uuid');
$table->uuid('batch_id');
$table->string('family_hash')->nullable();
$table->boolean('should_display_on_index')->default(true);
$table->string('type', 20);
$table->longText('content');
$table->dateTime('created_at')->nullable();
$table->unique('uuid');
$table->index('batch_id');
$table->index('family_hash');
$table->index('created_at');
$table->index(['type', 'should_display_on_index']);
});
$schema->create('telescope_entries_tags', function (Blueprint $table) {
$table->uuid('entry_uuid');
$table->string('tag');
$table->primary(['entry_uuid', 'tag']);
$table->index('tag');
$table->foreign('entry_uuid')
->references('uuid')
->on('telescope_entries')
->onDelete('cascade');
});
$schema->create('telescope_monitoring', function (Blueprint $table) {
$table->string('tag')->primary();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
$schema = Schema::connection($this->getConnection());
$schema->dropIfExists('telescope_entries_tags');
$schema->dropIfExists('telescope_entries');
$schema->dropIfExists('telescope_monitoring');
}
};

View File

@ -0,0 +1,32 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('failed_jobs', function (Blueprint $table) {
$table->id();
$table->string('uuid')->unique();
$table->text('connection');
$table->text('queue');
$table->longText('payload');
$table->longText('exception');
$table->timestamp('failed_at')->useCurrent();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('failed_jobs');
}
};

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

View File

@ -0,0 +1,5 @@
{
"/app.js": "/app.js?id=7049e92a398e816f8cd53a915eaea592",
"/app-dark.css": "/app-dark.css?id=b11fa9a28e9d3aeb8c92986f319b3c44",
"/app.css": "/app.css?id=b3ccfbe68f24cff776f83faa8dead721"
}

View File

@ -0,0 +1,37 @@
<style>
@media(max-width: 960px ){
#roman{
display: none;
}
#small{
display: block !important;
}
}
</style>
<span id="small" style="display: none;">
Consultancy Project 1
</span>
<pre id="roman" style="font-family: monospace; position: absolute; top: 50%; left: 50%; transform: translate3d(-50%, -50%, 0); " title="Consultancy Project 1">
.oooooo. oooo .
d8P' `Y8b `888 .o8
888 .ooooo. ooo. .oo. .oooo.o oooo oooo 888 .o888oo .oooo. ooo. .oo. .ooooo. oooo ooo
888 d88' `88b `888P"Y88b d88( "8 `888 `888 888 888 `P )88b `888P"Y88b d88' `"Y8 `88. .8'
888 888 888 888 888 `"Y88b. 888 888 888 888 .oP"888 888 888 888 `88..8'
`88b ooo 888 888 888 888 o. )88b 888 888 888 888 . d8( 888 888 888 888 .o8 `888'
`Y8bood8P' `Y8bod8P' o888o o888o 8""888P' `V88V"V8P' o888o "888" `Y888""8o o888o o888o `Y8bod8P' .8'
.o..P'
`Y8P'
ooooooooo. o8o . .o
`888 `Y88. `"' .o8 o888
888 .d88' oooo d8b .ooooo. oooo .ooooo. .ooooo. .o888oo 888
888ooo88P' `888""8P d88' `88b `888 d88' `88b d88' `"Y8 888 888
888 888 888 888 888 888ooo888 888 888 888
888 888 888 888 888 888 .o 888 .o8 888 . 888
o888o d888b `Y8bod8P' 888 `Y8bod8P' `Y8bod8P' "888" o888o
888
.o. 88P
`Y888P
</pre>

File diff suppressed because one or more lines are too long

View File

@ -4,13 +4,15 @@ use Illuminate\Support\Facades\Route;
use App\Scraper\Edomizil;
Route::get('/', function () {
return view('welcome');
return view('index');
});
Route::get('/properties', function () {
Edomizil::getProperties();
Edomizil::dispatchPropertyJobs();
});
Route::get('/occupancy', function () {
Edomizil::getOccupancies();
//$properties = Edomizil::getAllOccupancies();
Edomizil::dispatchPropertyDataJobs();
});