1:- module(tus, [set_tus_options/1, % +Options
2 % server
3 tus_dispatch/1, % +Request
4 tus_dispatch/2, % +Options, +Request
5 tus_dispatch/3, % +Method, +Options, +Request
6
7 % client
8 tus_options/3, % +Endpoint_URI, -Tus_Options, +Options
9
10 tus_upload/3, % +File, +Endpoint_URI, -Resource_URI
11 tus_upload/4, % +File, +Endpoint_URI, -Resource_URI,
12 % +Options
13 tus_resume/3, % +File, +Endpoint_URI, ?Resource_URI
14 tus_resume/4, % +File, +Endpoint_URI, ?Resource_URI,
15 % +Options
16 tus_delete/3, % +Resource_URI, +TUS_Options, +Options
17
18 tus_uri_resource/2, % +URI, -Resource
19
20 tus_resource_path/3 % +Resource, -Path, +Options
21 ]).
45:- use_module(library(http/http_client)). 46:- use_module(library(http/http_header)). 47:- use_module(library(http/thread_httpd)). 48 49/* parsing tools */ 50:- use_module(tus/parse). 51:- use_module(tus/utilities). 52 53:- use_module(library(apply)). 54:- use_module(library(crypto)). 55:- use_module(library(debug)). 56:- use_module(library(error)). 57:- use_module(library(lists)). 58:- use_module(library(option)). 59:- use_module(library(readutil)). 60:- use_module(library(md5)). 61:- use_module(library(when)). 62:- use_module(library(yall)). 63 64:- use_module(library(plunit)). 65 66/* Options */ 67:- meta_predicate valid_options( , ). 68valid_options(Options, Pred) :- 69 must_be(list, Options), 70 verify_options(Options, Pred). 71 72verify_options([], _). 73verify_options([H|T], Pred) :- 74 ( call(Pred, H) 75 -> verify_options(T, Pred) 76 ; throw(error(domain_error(Pred, H), _)) 77 ).
90set_tus_options(Options) :- 91 retract_tus_dynamics, 92 valid_options(Options, global_tus_option), 93 create_prolog_flag(tus_options, Options, []). 94 95global_tus_option(tus_storage_path(_Path)). 96global_tus_option(tus_max_size(Size)) :- 97 must_be(positive_integer, Size). 98global_tus_option(tus_client_chunk_size(Size)) :- 99 must_be(positive_integer, Size). 100global_tus_option(tus_expiry_seconds(Seconds)) :- 101 must_be(positive_integer, Seconds). 102 103tus_default_options( 104 [ 105 tus_max_size(17_179_869_184), 106 tus_client_chunk_size(16_777_216), 107 tus_expiry_seconds(86400), 108 tus_storage_path(_Dummy) 109 ] 110). 111 112tus_merged_options(Options) :- 113 tus_default_options(Defaults), 114 ( current_prolog_flag(tus_options, User) 115 -> true 116 ; User = []), 117 merge_options(User, Defaults, Options). 118 119get_tus_option(Option) :- 120 tus_merged_options(Options), 121 option(Option, Options). 122 123:- dynamic tus_storage_path_dynamic/1. 124 125retract_tus_dynamics :- 126 retractall(tus_storage_path_dynamic(_)). 127 128/* Tus server constants */ 129tus_version_list('1.0.0'). 130 131tus_max_size(Size) :- 132 get_tus_option(tus_max_size(Size)). 133 134tus_client_chunk_size(Size) :- 135 get_tus_option(tus_client_chunk_size(Size)). 136 137tus_expiry_seconds(Seconds) :- 138 get_tus_option(tus_expiry_seconds(Seconds)). 139 140tus_extension(creation). 141tus_extension(expiration). 142tus_extension(checksum). 143tus_extension(termination). 144 145tus_extension_list(Atom) :- 146 findall(Extension, 147 tus_extension(Extension), 148 Extensions), 149 atomic_list_concat(Extensions, ',', Atom). 150 151% In precedence order 152tus_checksum_algorithm(sha1). 153tus_checksum_algorithm(md5). 154 155tus_checksum_algorithm_list(Atom) :- 156 findall(Extension, 157 tus_checksum_algorithm(Extension), 158 Extensions), 159 atomic_list_concat(Extensions, ',', Atom). 160 161tus_storage_path(Path, Options) :- 162 memberchk(tus_storage_path(Pre_Path), Options), 163 terminal_slash(Pre_Path, Path), 164 !. 165tus_storage_path(Path, _Options) :- 166 tus_storage_path_dynamic(Path), 167 !. 168tus_storage_path(Path, _Options) :- 169 current_prolog_flag(tus_options, Options), 170 memberchk(tus_storage_path(Pre_Path), Options), 171 terminal_slash(Pre_Path, Path), 172 assertz(tus_storage_path_dynamic(Path)), 173 !. 174tus_storage_path(Temp, _Options) :- 175 random_file('tus_storage', Temp_File), 176 make_directory(Temp_File), 177 terminal_slash(Temp_File,Temp), 178 assertz(tus_storage_path_dynamic(Temp)). 179 180accept_tus_version(Version) :- 181 member(Version, ['1.0.0']). 182 183expiration_timestamp(Expiry) :- 184 get_time(Time), 185 tus_expiry_seconds(Seconds), 186 Expiry is Time + Seconds. 187 188algorithm_checksum_string(md5, Checksum, String) :- 189 md5_hash(String, Checksum, []). 190algorithm_checksum_string(sha1, Checksum, String) :- 191 crypto_data_hash(String, Checksum_Pre, [algorithm(sha1), 192 encoding(octet)]), 193 Checksum_Pre = Checksum. 194 195algorithm_checksum_string_value(Algorithm, Checksum, String, Value) :- 196 when( 197 ( ground(Algorithm), 198 ground(Checksum) 199 ; ground(Value)), 200 atomic_list_concat([Algorithm, Checksum], ' ', Value) 201 ), 202 algorithm_checksum_string(Algorithm, Checksum, String).
207tus_checksum(Upload_Checksum,String) :- 208 algorithm_checksum_string_value(_Algorithm, _Checksum, String, Upload_Checksum). 209 210tus_algorithm_supported(Upload_Checksum) :- 211 atomic_list_concat([Algorithm, _], ' ', Upload_Checksum), 212 tus_checksum_algorithm(Algorithm). 213 214tus_generate_checksum_header(String, Header, Tus_Options) :- 215 memberchk(tus_checksum_algorithm(Algorithms), Tus_Options), 216 tus_checksum_algorithm(Algorithm), % in this order of precedence 217 member(Algorithm, Algorithms), 218 !, 219 algorithm_checksum_string_value(Algorithm, _, String, Upload_Checksum), 220 Header = [request_header('Upload-Checksum'=Upload_Checksum)]. 221tus_generate_checksum_header(_, Header, _) :- 222 Header = []. 223 224% Should probably also be an option. 225tus_max_retries(3). 226 227check_tus_creation(Tus_Options) :- 228 memberchk(tus_extension(Extensions), Tus_Options), 229 memberchk(creation, Extensions). 230 231check_tus_termination(Tus_Options) :- 232 memberchk(tus_extension(Extensions), Tus_Options), 233 memberchk(termination, Extensions). 234 235/* 236File store manipulation utilities. 237*/ 238tus_resource_name(File, Name) :- 239 md5_hash(File, Name, []). 240 241tus_resource_base_path(Resource, Path, Options) :- 242 tus_storage_path(Storage, Options), 243 option(domain(Domain),Options,'xanadu'), 244 atomic_list_concat([Storage, Domain, '.', Resource], Path). 245 246tus_resource_suffix('completed').
260tus_resource_path(Resource, Path, Options) :- 261 tus_resource_base_path(Resource, Base_Path, Options), 262 tus_resource_suffix(Completed), 263 atomic_list_concat([Base_Path, '/',Completed], Path). 264 265tus_resource_deleted_path(Resource, Deleted_Path, Options) :- 266 tus_resource_base_path(Resource, Path, Options), 267 atomic_list_concat([Path, '.deleted'], Deleted_Path). 268 269tus_offset_suffix('offset'). 270 271tus_offset_path(Resource, Path, Options) :- 272 tus_resource_base_path(Resource, RPath, Options), 273 tus_offset_suffix(Offset), 274 atomic_list_concat([RPath, '/', Offset], Path). 275 276tus_size_suffix('size'). 277 278tus_size_path(Resource, Path, Options) :- 279 tus_resource_base_path(Resource, RPath, Options), 280 tus_size_suffix(Size), 281 atomic_list_concat([RPath, '/', Size], Path). 282 283tus_upload_suffix('upload'). 284 285tus_upload_path(Resource, Path, Options) :- 286 tus_resource_base_path(Resource, RPath, Options), 287 tus_upload_suffix(Upload), 288 atomic_list_concat([RPath, '/', Upload], Path). 289 290tus_lock_suffix('lock'). 291 292tus_lock_path(Resource, Path, Options) :- 293 tus_resource_base_path(Resource, RPath, Options), 294 tus_lock_suffix(Lock), 295 atomic_list_concat([RPath, '/', Lock], Path). 296 297tus_expiry_suffix('expiry'). 298 299tus_expiry_path(Resource, Path, Options) :- 300 tus_resource_base_path(Resource, RPath, Options), 301 tus_expiry_suffix(Offset), 302 atomic_list_concat([RPath, '/', Offset], Path). 303 304tus_metadata_suffix('metadata'). 305 306tus_metadata_path(Resource, Path, Options) :- 307 tus_resource_base_path(Resource, RPath, Options), 308 tus_metadata_suffix(Metadata), 309 atomic_list_concat([RPath, '/', Metadata], Path). 310 311 312% turns exception into failure. 313try_make_directory(Directory) :- 314 catch( 315 make_directory(Directory), 316 error(existence_error(directory,_),_), 317 fail). 318 319:- meta_predicate fail_on_missing_file( ). 320fail_on_missing_file(Goal) :- 321 catch( 322 call(Goal), 323 error(existence_error(source_sink,_),_), 324 fail). 325 326path_contents(Offset_File, Offset_String) :- 327 fail_on_missing_file( 328 read_file_to_string(Offset_File, Offset_String, []) 329 ). 330 331resource_offset(Resource, Offset, Options) :- 332 tus_offset_path(Resource, Offset_File, Options), 333 path_contents(Offset_File, Offset_String), 334 atom_number(Offset_String, Offset). 335 336set_resource_offset(Resource, Offset, Options) :- 337 setup_call_cleanup( 338 ( tus_offset_path(Resource, Offset_Path, Options), 339 open(Offset_Path, write, OP)), 340 format(OP, "~d", [Offset]), 341 close(OP) 342 ). 343 344resource_size(Resource, Size, Options) :- 345 tus_size_path(Resource, Size_File, Options), 346 path_contents(Size_File, Size_String), 347 atom_number(Size_String, Size). 348 349set_resource_size(Resource, Size, Options) :- 350 setup_call_cleanup( 351 ( tus_size_path(Resource, Size_Path, Options), 352 open(Size_Path, write, SP)), 353 format(SP, "~d", [Size]), 354 close(SP) 355 ). 356 357resource_expiry(Resource, Expiry, Options) :- 358 tus_expiry_path(Resource, Expiry_File, Options), 359 path_contents(Expiry_File, Expiry_String), 360 atom_number(Expiry_String, Expiry). 361 362set_resource_expiry(Resource, Size, Options) :- 363 setup_call_cleanup( 364 ( tus_expiry_path(Resource, Size_Path, Options), 365 open(Size_Path, write, SP)), 366 format(SP, "~q", [Size]), 367 close(SP) 368 ). 369 370resource_metadata(Resource, Metadata, Options) :- 371 tus_metadata_path(Resource, Metadata_File, Options), 372 path_contents(Metadata_File, Metadata_String), 373 read_term_from_atom(Metadata_String, Metadata, []). 374 375set_resource_metadata(Resource, Metadata, Options) :- 376 setup_call_cleanup( 377 ( tus_metadata_path(Resource, Metadata_Path, Options), 378 open(Metadata_Path, write, OP)), 379 format(OP, "~q", [Metadata]), 380 close(OP) 381 ). 382 383 384% member(Status, [exists, expires(Expiry)]) 385create_file_resource(Metadata, Size, Name, Status, Options) :- 386 ( memberchk(filename-File, Metadata) 387 -> true 388 ; random_string(File) 389 ), 390 391 tus_resource_name(File, Name), 392 tus_resource_base_path(Name, Path, Options), 393 ( try_make_directory(Path) 394 -> % File 395 setup_call_cleanup( 396 ( tus_lock_path(Name, Lock_Path, Options), 397 open(Lock_Path, write, LP, [lock(exclusive)]) % Get an exclusive lock 398 ), 399 setup_call_cleanup( 400 ( tus_upload_path(Name,File_Path, Options), 401 open(File_Path, write, FP) 402 ), 403 ( % Size 404 set_resource_size(Name, Size, Options), 405 406 % Offset 407 set_resource_offset(Name, 0, Options), 408 409 % Expires 410 expiration_timestamp(Expiry), 411 set_resource_expiry(Name, Expiry, Options), 412 413 % Metadata 414 set_resource_metadata(Name, Metadata, Options) 415 ), 416 close(FP) 417 ), 418 close(LP) 419 ), 420 Status=expires(Expiry) 421 ; Status=exists 422 ). 423 424patch_resource(Name, Patch, Offset, Length, New_Offset, Options) :- 425 426 % sanity check 427 ( string_length(Patch, Length) 428 -> true 429 ; throw(error(bad_length(Length)))), 430 431 ( resource_offset(Name, Offset, Options) 432 -> true 433 ; throw(error(bad_offset(Offset)))), 434 435 setup_call_cleanup( 436 ( tus_lock_path(Name, Lock_Path, Options), 437 open(Lock_Path, write, LP, [lock(exclusive)]) % Get an exclusive lock 438 ), 439 ( setup_call_cleanup( 440 ( tus_upload_path(Name, Upload_Path, Options), 441 open(Upload_Path, update, UP, [encoding(octet)]) 442 ), 443 ( seek(UP, Offset, bof, _), 444 format(UP, "~s", [Patch]), 445 New_Offset is Offset + Length, 446 447 set_resource_offset(Name, New_Offset, Options) 448 ), 449 close(UP) 450 ), 451 resource_size(Name, Size, Options), 452 ( Size = New_Offset 453 -> tus_resource_path(Name, Resource_Path, Options), 454 rename_file(Upload_Path, Resource_Path) 455 ; true 456 ) 457 ), 458 close(LP) 459 ). 460 461delete_resource(Resource, Options) :- 462 tus_resource_base_path(Resource, Path, Options), 463 tus_resource_deleted_path(Resource, Deleted, Options), 464 rename_file(Path, Deleted), 465 directory_files(Deleted, All_Files), 466 exclude([X]>>(member(X,['.', '..'])), All_Files, Files), 467 forall( 468 member(File, Files), 469 ( atomic_list_concat([Deleted, '/', File],Full_Path), 470 delete_file(Full_Path))), 471 delete_directory(Deleted). 472 473tus_client_effective_chunk_size(Options, Chunk) :- 474 memberchk(tus_max_size(Max), Options), 475 tus_client_chunk_size(Size), 476 Chunk is min(Max,Size). 477 478chunk_directive_(Length, Chunk_Size, [Length-0]) :- 479 Length =< Chunk_Size, 480 !. 481chunk_directive_(Length, Chunk_Size, [Chunk_Size-New_Length|Directive]) :- 482 Length > Chunk_Size, 483 !, 484 New_Length is Length - Chunk_Size, 485 chunk_directive_(New_Length, Chunk_Size, Directive). 486 487/* 488 * Generators for chunk offsets 489 */ 490chunk_directive(Length, Chunk_Size, Current_Offset, Current_Chunk) :- 491 chunk_directive(0, Length, Chunk_Size, Current_Offset, Current_Chunk). 492 493chunk_directive(Offset, Length, Chunk_Size, Offset, Current_Chunk) :- 494 Offset < Length, 495 Current_Chunk is min(Chunk_Size, Length - Offset). 496chunk_directive(Offset, Length, Chunk_Size, Current_Offset, Current_Chunk) :- 497 Next_Chunk is min(Chunk_Size, Length - Offset), 498 Next_Offset is Offset + Next_Chunk, 499 Next_Offset < Length, 500 chunk_directive(Next_Offset, Length, Chunk_Size, Current_Offset, Current_Chunk).
507tus_uri_resource(URI, Resource) :- 508 split_string(URI, '/', '', List), 509 last(List, Resource). 510 511terminal_slash(Atom, Slashed) :- 512 split_string(Atom, '/', '', List), 513 last(List, Last), 514 ( Last = "" 515 -> Atom = Slashed 516 ; atomic_list_concat([Atom, '/'], Slashed)). 517 518resumable_endpoint(_, Name, Endpoint, Options) :- 519 memberchk(resumable_endpoint_base(Pre_Base), Options), 520 !, 521 terminal_slash(Pre_Base,Base), 522 format(atom(Endpoint), "~s~s",[Base,Name]). 523resumable_endpoint(Request, Name, Endpoint, _Options) :- 524 memberchk(protocol(Protocol),Request), 525 memberchk(host(Server),Request), 526 ( memberchk(port(Port),Request) 527 -> true 528 ; Port = 80 529 ), 530 memberchk(request_uri(Pre_Base),Request), 531 terminal_slash(Pre_Base,Base), 532 format(atom(Endpoint), "~s://~s:~d~s~s", [Protocol,Server,Port,Base,Name]). 533 534% This is a terrible way to get the output stream... 535% something is broken - should be set in current_output 536http_output_stream(Request, Out) :- 537 memberchk(pool(client(_,_,_,Out)), Request). 538 539format_headers(_, []). 540format_headers(Out, [Term|Rest]) :- 541 Term =.. [Atom,Value], 542 format(Out, "~s: ~w~n", [Atom, Value]), 543 format_headers(Out, Rest). 544 545format_custom_response(Out, checksum_mismatch) :- 546 format(Out, "HTTP/1.1 460 Checksum Mismatch~n",[]). 547format_custom_response(Out, conflict) :- 548 format(Out, "HTTP/1.1 409 Conflict~n",[]). 549format_custom_response(Out, gone) :- 550 format(Out, "HTTP/1.1 410 Gone~n",[]). 551 552custom_status_reply(Custom_Response, Out, Headers) :- 553 format_custom_response(Out,Custom_Response), 554 format_headers(Out,Headers), 555 format(Out,"\r\n",[]). 556 557status_code(created,201). 558status_code(no_content,204). 559status_code(bad_request,400). 560status_code(forbidden,403). 561status_code(not_found,404). 562status_code(conflict,409). 563status_code(gone,410). 564status_code(unsupported_media,415). 565status_code(bad_checksum,460). 566 567/* 568 * Server dispatch 569 */
/
577tus_dispatch(Request) :-
578 tus_dispatch([],Request).
Should be callable from http_handler/3 with something along the lines of:
` http_handler(root(files), tus_dispatch,
[ methods([options,head,post,patch]),
prefix
])
`
599tus_dispatch(Options,Request) :-
600 ( memberchk('X-HTTP-Method-Override'(Method), Request)
601 -> true
602 ; memberchk(method(Method),Request)),
603 tus_dispatch(Method,Options,Request).
616tus_dispatch(options,_Options,Request) :- 617 % Options 618 !, 619 tus_version_list(Version_List), 620 tus_max_size(Max_Size), 621 tus_extension_list(Extension_List), 622 tus_checksum_algorithm_list(Algorithm_List), 623 http_output_stream(Request, Out), 624 http_status_reply(no_content, Out, 625 ['Tus-Resumable'('1.0.0'), 626 'Tus-Version'(Version_List), 627 'Tus-Max-Size'(Max_Size), 628 'Tus-Checksum-Algorithm'(Algorithm_List), 629 'Tus-Extension'(Extension_List) 630 ], 631 204). 632tus_dispatch(post,Options,Request) :- 633 % Create 634 !, 635 memberchk(upload_length(Length_Atom),Request), 636 atom_number(Length_Atom, Length), 637 638 memberchk(upload_metadata(Metadata_Atom),Request), 639 debug(tus, "~q", [Metadata_Atom]), 640 parse_upload_metadata(Metadata_Atom, Metadata), 641 642 memberchk(tus_resumable(Version),Request), 643 accept_tus_version(Version), 644 645 create_file_resource(Metadata, Length, Name, Status, Options), 646 647 resumable_endpoint(Request, Name, Endpoint, Options), 648 649 http_output_stream(Request, Out), 650 ( Status = exists 651 -> custom_status_reply(conflict, Out, 652 ['Tus-Resumable'('1.0.0'), 653 'Location'(Endpoint)]) 654 ; Status = expires(Expiry), 655 http_timestamp(Expiry, Expiry_Date), 656 http_status_reply(created(Endpoint), Out, 657 ['Tus-Resumable'('1.0.0'), 658 'Upload-Expires'(Expiry_Date)], 659 _Code2) 660 ). 661tus_dispatch(head,Options,Request) :- 662 % Find position 663 !, 664 memberchk(request_uri(URI), Request), 665 tus_uri_resource(URI, Resource), 666 http_output_stream(Request, Out), 667 668 ( resource_offset(Resource, Offset, Options), 669 resource_size(Resource, Size, Options) 670 -> http_reply(bytes('application/offset+octet-stream',""), Out, 671 ['Tus-Resumable'('1.0.0'), 672 'Upload-Offset'(Offset), 673 'Upload-Length'(Size), 674 'Cache-Control'('no-store')]) 675 ; http_status_reply(not_found(URI), Out, 676 ['Tus-Resumable'('1.0.0')], 677 _Code) 678 ). 679tus_dispatch(patch,Options,Request) :- 680 % Patch next bit 681 !, 682 memberchk(request_uri(URI),Request), 683 memberchk(content_length(Length),Request), 684 685 memberchk(upload_offset(Offset_Atom),Request), 686 atom_number(Offset_Atom, Offset), 687 688 http_read_data(Request, Patch, []), 689 690 http_output_stream(Request, Out), 691 ( memberchk(upload_checksum(Upload_Checksum),Request), 692 \+ tus_checksum(Upload_Checksum,Patch) 693 % We have a checksum to check and it doesn't check out. 694 -> ( \+ tus_algorithm_supported(Upload_Checksum) 695 % Because the algorithm is unsupported 696 -> http_status_reply(bad_request('Algorithm Unsupported'), Out, 697 ['Tus-Resumable'('1.0.0')], 698 _Code) 699 % Because the checksum is wrong 700 ; custom_status_reply(checksum_mismatch, Out, 701 ['Tus-Resumable'('1.0.0')]) 702 ) 703 % No checksum, or it checks out. 704 ; tus_uri_resource(URI, Resource), 705 ( resource_expiry(Resource, Expiry, Options) 706 -> http_timestamp(Expiry, Expiry_Date), 707 ( get_time(Time), 708 debug(tus, "Time: ~q Expiry: ~q~n", [Time, Expiry]), 709 Time > Expiry 710 -> custom_status_reply(gone, Out, 711 ['Tus-Resumable'('1.0.0'), 712 'Upload-Expires'(Expiry_Date)]) 713 ; patch_resource(Resource, Patch, Offset, Length, New_Offset, Options), 714 http_status_reply(no_content, Out, 715 ['Tus-Resumable'('1.0.0'), 716 'Upload-Expires'(Expiry_Date), 717 'Upload-Offset'(New_Offset)], 718 _Code)) 719 ; http_status_reply(not_found(URI), Out, 720 ['Tus-Resumable'('1.0.0')], 721 _Code) 722 ) 723 ). 724tus_dispatch(delete,Options,Request) :- 725 % Delete 726 memberchk(request_uri(URI),Request), 727 728 tus_uri_resource(URI, Resource), 729 730 http_output_stream(Request, Out), 731 732 ( delete_resource(Resource, Options) 733 -> http_status_reply(no_content, Out, 734 ['Tus-Resumable'('1.0.0')], 735 _Code) 736 ; http_status_reply(not_found(URI), Out, 737 ['Tus-Resumable'('1.0.0')], 738 _Code) 739 ). 740 741/* 742 Client implementation 743 744*/ 745 746tus_process_options([], []). 747tus_process_options([tus_checksum_algorithm(X)|Rest_In],[tus_checksum_algorithm(Y)|Rest_Out]) :- 748 !, 749 atomic_list_concat(Y, ',', X), 750 tus_process_options(Rest_In, Rest_Out). 751tus_process_options([tus_extension(X)|Rest_In],[tus_extension(Y)|Rest_Out]) :- 752 !, 753 atomic_list_concat(Y, ',', X), 754 tus_process_options(Rest_In, Rest_Out). 755tus_process_options([tus_max_size(X)|Rest_In],[tus_max_size(Y)|Rest_Out]) :- 756 !, 757 atom_number(X,Y), 758 tus_process_options(Rest_In, Rest_Out). 759tus_process_options([X|Rest_In],[X|Rest_Out]) :- 760 tus_process_options(Rest_In, Rest_Out). 761 762tus_options(Endpoint, Tus_Options, Options) :- 763 http_client:headers_option(Options, Options1, Headers), 764 option(reply_header(Headers), Options, _), 765 http_client:http_open(Endpoint, In, [method(options), 766 status_code(204) 767 |Options1]), 768 close(In), 769 tus_process_options(Headers, Tus_Options). 770 771tus_create(Endpoint, File, Length, Resource, Tus_Options, Options) :- 772 tus_create(Endpoint, File, Length, Resource, _, Tus_Options, Options). 773 774tus_create(Endpoint, File, Length, Resource, Reply_Header, Tus_Options, Options) :- 775 ( check_tus_creation(Tus_Options) 776 -> true 777 ; throw(error(no_creation_extention(Endpoint), _))), 778 779 size_file(File, Length), 780 parse_upload_metadata(Metadata,[filename-File]), 781 http_get(Endpoint, Response, [ 782 method(post), 783 request_header('Upload-Length'=Length), 784 request_header('Upload-Metadata'=Metadata), 785 request_header('Tus-Resumable'='1.0.0'), 786 content_length(0), 787 reply_header(Reply_Header), 788 status_code(Code) 789 |Options 790 ]), 791 ( status_code(Status,Code) 792 -> ( Status = created 793 -> memberchk(location(Resource), Reply_Header) 794 ; Status = conflict % file already exists 795 -> throw(error(file_already_exists(File), _)) 796 ; throw(error(unhandled_status_code(Code,Response),_))) 797 ; throw(error(unhandled_status_code(Code,Response),_)) 798 ). 799 800tus_patch(Endpoint, File, Chunk, Position, Tus_Options, Options) :- 801 tus_patch(Endpoint, File, Chunk, Position, _Reply_Header, Tus_Options, Options). 802 803tus_patch(Endpoint, File, Chunk, Position, Reply_Header, Tus_Options, Options) :- 804 tus_max_retries(Max_Retries), 805 between(0,Max_Retries,_Tries), 806 tus_patch_(Endpoint, File, Chunk, Position, Reply_Header, Tus_Options, Options), 807 !. 808tus_patch(Endpoint, _, _, _, _, _, _) :- 809 tus_max_retries(Max_Retries), 810 throw(error(exceeded_max_retries(Endpoint,Max_Retries),_)). 811 812tus_patch_(Endpoint, File, Chunk, Position, Reply_Header, Tus_Options, Options) :- 813 setup_call_cleanup( 814 open(File, read, Stream, [encoding(octet)]), 815 ( seek(Stream, Position, bof, _), 816 read_string(Stream, Chunk, String), 817 tus_generate_checksum_header(String, Header, Tus_Options), 818 append(Options,Header,HdrExtra), 819 http_get(Endpoint, Response, [ 820 method(patch), 821 post(bytes('application/offset+octet-stream', String)), 822 request_header('Upload-Offset'=Position), 823 request_header('Tus-Resumable'='1.0.0'), 824 reply_header(Reply_Header), 825 status_code(Code) 826 |HdrExtra 827 ]) 828 ), 829 close(Stream) 830 ), 831 ( status_code(Status,Code) 832 -> ( Status = no_content % patched 833 -> true 834 ; Status = bad_checksum 835 -> fail % (i.e. retry) 836 ; Status = bad_request % No request algorithm 837 -> throw(error(bad_request(Endpoint), _)) 838 ; Status = not_found 839 -> throw(error(not_found(Endpoint), _)) 840 ; Status = gone 841 -> throw(error(gone(Endpoint),_)) 842 ; Status = forbidden 843 -> throw(error(forbidden(Endpoint),_)) 844 ; Status = unsupported_media 845 -> throw(error(unsupported_media,_)) 846 ; throw(error(unhandled_status_code(Code,Response),_)) 847 ) 848 ; throw(error(unhandled_status_code(Code,Response),_)) 849 ). 850 851tus_head(Resource_URL, Offset, Length, Options) :- 852 tus_head(Resource_URL, Offset, Length, _Reply_Header, Options). 853 854tus_head(Resource_URL, Offset, Length, Reply_Header, Options) :- 855 http_get(Resource_URL, _, [ 856 method(head), 857 request_header('Tus-Resumable'='1.0.0'), 858 reply_header(Reply_Header), 859 status_code(200) 860 |Options 861 ]), 862 memberchk(upload_offset(Offset_Atom),Reply_Header), 863 atom_number(Offset_Atom, Offset), 864 ( memberchk(upload_length(Length_Atom),Reply_Header) 865 -> atom_number(Length_Atom, Length) 866 ; Length = unknown 867 ). 868 869tus_delete(Resource_URL, Tus_Options, Options) :- 870 tus_delete(Resource_URL, _Reply_Header, Tus_Options, Options). 871 872tus_delete(Resource_URL, Reply_Header, Tus_Options, Options) :- 873 ( check_tus_termination(Tus_Options) 874 -> true 875 ; throw(error(no_termination_extention(Resource_URL), _))), 876 877 http_get(Resource_URL, Response, [ 878 method(delete), 879 request_header('Tus-Resumable'='1.0.0'), 880 reply_header(Reply_Header), 881 status_code(Code) 882 |Options 883 ]), 884 885 ( status_code(Status, Code) 886 -> ( Status = no_content % deleted 887 -> true 888 ; memberchk(Status, [not_found, gone]) 889 -> throw(error(resource_does_not_exist(Resource_URL), _)) 890 ; Status = forbidden 891 -> throw(error(forbidden(Resource_URL),_)) 892 ; throw(error(unhandled_status_code(Code,Response),_)) 893 ) 894 ; throw(error(unhandled_status_code(Code,Response),_)) 895 ).
907tus_upload(File, Endpoint, Resource_URL, Options) :- 908 tus_options(Endpoint, Tus_Options, Options), 909 tus_create(Endpoint, File, Length, Resource_URL, Tus_Options, Options), 910 tus_client_effective_chunk_size(Tus_Options, Chunk_Size), 911 forall( 912 chunk_directive(Length, Chunk_Size, Position, Chunk), 913 ( debug(tus, "Chunk: ~q Position: ~q~n", [Chunk, Position]), 914 tus_patch(Resource_URL, File, Chunk, Position, Tus_Options, Options) 915 ) 916 ). 917 918tus_upload(File, Endpoint, Resource_URL) :- 919 tus_upload(File, Endpoint, Resource_URL, []).
931tus_resume(File, Endpoint, Resource_URL, Options) :- 932 tus_options(Endpoint, Tus_Options, Options), 933 tus_head(Resource_URL, Offset, Length, Options), 934 tus_client_effective_chunk_size(Tus_Options, Chunk_Size), 935 forall( 936 chunk_directive(Offset, Length, Chunk_Size, Position, Chunk), 937 ( debug(tus, "Chunk: ~q Position: ~q~n", [Chunk, Position]), 938 tus_patch(Resource_URL, File, Chunk, Position, Tus_Options, Options) 939 ) 940 ). 941 942tus_resume(File, Endpoint, Resource_URL) :- 943 tus_resume(File, Endpoint, Resource_URL, []). 944 945/* Tests */ 946 947:- begin_tests(tus). 948:- use_module(library(random)). 949 950spawn_server(URL, Port, Options) :- 951 random_between(49152, 65535, Port), 952 http_server(tus_dispatch(Options), [port(Port), workers(1)]), 953 format(atom(URL), 'http://127.0.0.1:~d/files', [Port]). 954 955kill_server(Port) :- 956 http_stop_server(Port,[]). 957 958test(send_file, [ 959 setup((set_tus_options([tus_client_chunk_size(4)]), 960 random_file(tus_storage_test, Path), 961 make_directory(Path), 962 Options = [tus_storage_path(Path)], 963 spawn_server(URL, Port, Options))), 964 cleanup(kill_server(Port)) 965 ]) :- 966 967 random_file('example_txt_tus', File), 968 open(File, write, Stream), 969 Content = "asdf fdsa yes yes yes", 970 format(Stream, '~s', [Content]), 971 close(Stream), 972 973 tus_upload(File, URL, _Resource), 974 975 tus_resource_name(File, Name), 976 tus_resource_path(Name, Resource, Options), 977 read_file_to_string(Resource, Result, []), 978 979 Result = Content. 980 981test(send_and_delete_file, [ 982 setup((set_tus_options([tus_client_chunk_size(4)]), 983 random_file(tus_storage_test, Path), 984 make_directory(Path), 985 Options = [tus_storage_path(Path)], 986 spawn_server(URL, Port, Options))), 987 cleanup(kill_server(Port)) 988 ]) :- 989 990 random_file('example_txt_tus', File), 991 open(File, write, Stream), 992 Content = "asdf fdsa yes yes yes", 993 format(Stream, '~s', [Content]), 994 close(Stream), 995 996 tus_upload(File, URL, Resource), 997 998 tus_resource_name(File, Name), 999 tus_resource_path(Name, Resource_Path, Options), 1000 read_file_to_string(Resource_Path, _Result, []), 1001 1002 tus_options(URL, Tus_Options, []), 1003 tus_delete(Resource, Tus_Options, Options), 1004 tus_resource_base_path(Resource, Base_Path, Options), 1005 \+ exists_directory(Base_Path). 1006 1007test(check_expiry, [ 1008 setup((set_tus_options([tus_client_chunk_size(4)]), 1009 random_file(tus_storage_test, Path), 1010 make_directory(Path), 1011 Options = [tus_storage_path(Path)], 1012 spawn_server(URL, Port, Options))), 1013 cleanup(kill_server(Port)) 1014 ]) :- 1015 1016 random_file('example_txt_tus', File), 1017 open(File, write, Stream), 1018 Content = "asdf fdsa yes yes yes", 1019 format(Stream, '~s', [Content]), 1020 close(Stream), 1021 1022 tus_options(URL, Tus_Options, []), 1023 tus_create(URL, File, _Length, Resource, Reply_Header_Create, []), 1024 % TODO: This should actually parse as RFC7231 1025 % and check the date is in the future. 1026 memberchk(upload_expires(_Date_String1), 1027 Reply_Header_Create), 1028 1029 tus_patch(Resource, File, 4, 0, Reply_Header_Patch, Tus_Options, []), 1030 memberchk(upload_expires(_Date_String2), 1031 Reply_Header_Patch). 1032 1033test(expired_reinitiated, [ 1034 setup((set_tus_options([tus_client_chunk_size(4), 1035 tus_expiry_seconds(1) 1036 ]), 1037 random_file(tus_storage_test, Path), 1038 make_directory(Path), 1039 Options = [tus_storage_path(Path)], 1040 spawn_server(URL, Port, Options))), 1041 cleanup(kill_server(Port)), 1042 error(gone(Resource),_) 1043 ]) :- 1044 1045 random_file('example_txt_tus', File), 1046 open(File, write, Stream), 1047 Content = "asdf fdsa yes yes yes", 1048 format(Stream, '~s', [Content]), 1049 close(Stream), 1050 1051 tus_options(URL, Tus_Options, []), 1052 tus_create(URL, File, _Length, Resource, _, []), 1053 sleep(1), 1054 tus_patch(Resource, File, 4, 0, Tus_Options, []). 1055 1056test(resume, [ 1057 setup((set_tus_options([tus_client_chunk_size(4)]), 1058 random_file(tus_storage_test, Path), 1059 make_directory(Path), 1060 Options = [tus_storage_path(Path)], 1061 spawn_server(URL, Port, Options))), 1062 cleanup(kill_server(Port)) 1063 ]) :- 1064 1065 random_file('example_txt_tus', File), 1066 open(File, write, Stream), 1067 Content = "asdf fdsa yes yes yes", 1068 format(Stream, '~s', [Content]), 1069 close(Stream), 1070 1071 tus_options(URL, Tus_Options, []), 1072 tus_create(URL, File, _Length, Resource_URL, _, []), 1073 tus_patch(Resource_URL, File, 4, 0, Tus_Options, []), 1074 1075 tus_resume(File, URL, Resource_URL), 1076 1077 tus_resource_name(File, Name), 1078 tus_resource_path(Name, Resource, Options), 1079 read_file_to_string(Resource, Result, []), 1080 1081 Result = Content. 1082 1083test(bad_checksum, [ 1084 setup((set_tus_options([tus_client_chunk_size(4)]), 1085 random_file(tus_storage_test, Path), 1086 make_directory(Path), 1087 Options = [tus_storage_path(Path)], 1088 spawn_server(URL, Port, Options))), 1089 cleanup(kill_server(Port)), 1090 error(exceeded_max_retries(Resource_URL,_Tries),_) 1091 ]) :- 1092 1093 random_file('example_txt_tus', File), 1094 open(File, write, Stream), 1095 Content = "something else for a change", 1096 format(Stream, '~s', [Content]), 1097 close(Stream), 1098 1099 tus_options(URL, Tus_Options, []), 1100 tus_create(URL, File, _Length, Resource_URL, _, []), 1101 tus_patch(Resource_URL, File, 4, 0, Tus_Options, 1102 [request_header('Upload-Checksum'='sha1 33fd0301077bc24fc6513513c71e288fcecc0c66')]). 1103 1104test(bad_checksum_algo, [ 1105 setup((set_tus_options([tus_client_chunk_size(4)]), 1106 random_file(tus_storage_test, Path), 1107 make_directory(Path), 1108 Options = [tus_storage_path(Path)], 1109 spawn_server(URL, Port, Options))), 1110 cleanup(kill_server(Port)), 1111 error(bad_request(_),_) 1112 ]) :- 1113 1114 random_file('example_txt_tus', File), 1115 open(File, write, Stream), 1116 Content = "something else for a change", 1117 format(Stream, '~s', [Content]), 1118 close(Stream), 1119 1120 tus_options(URL, Tus_Options, []), 1121 tus_create(URL, File, _Length, Resource_URL, _, []), 1122 tus_patch(Resource_URL, File, 4, 0, Tus_Options, 1123 [request_header('Upload-Checksum'='scrunchy asdffdsa')]). 1124 1125/* Authorization 1126 1127 A test example with domains. 1128 1129 */ 1130:- use_module(library(http/http_authenticate)). 1131 1132auth_table(me,pass,shangrila). 1133 Request, Username, Key) (:- 1135 memberchk(authorization(Text), Request), 1136 http_authorization_data(Text, basic(Username, Key)). 1137 Request,Organization) (:- 1139 fetch_authorization_data(Request, Username, Key_Codes), 1140 atom_codes(Key,Key_Codes), 1141 auth_table(Username, Key, Organization). 1142 1143:- meta_predicate auth_wrapper( , , ). 1144auth_wrapper(Goal,Options,Request) :- 1145 authorize(Request, Domain), 1146 call(Goal, [domain(Domain) | Options], Request). 1147 1148spawn_auth_server(URL, Port, Options) :- 1149 random_between(49152, 65535, Port), 1150 http_server(auth_wrapper(tus_dispatch, Options), [port(Port), workers(1)]), 1151 format(atom(URL), 'http://127.0.0.1:~d/files', [Port]). 1152 1153test(auth_test, [ 1154 setup((set_tus_options([tus_client_chunk_size(4), 1155 tus_expiry_seconds(1) 1156 ]), 1157 random_file(tus_storage_test, Path), 1158 make_directory(Path), 1159 Options = [tus_storage_path(Path)], 1160 spawn_auth_server(URL, Port, [tus_storage_path(Path)]))), 1161 cleanup(kill_server(Port)) 1162 ]) :- 1163 1164 random_file('example_txt_tus', File), 1165 open(File, write, Stream), 1166 Content = "asdf fdsa yes yes yes", 1167 format(Stream, '~s', [Content]), 1168 close(Stream), 1169 1170 tus_upload(File, URL, _Resource, [authorization(basic(me,pass))]), 1171 1172 tus_resource_name(File, Name), 1173 tus_resource_path(Name, Resource, [domain(shangrila) | Options]), 1174 read_file_to_string(Resource, Result, []), 1175 1176 Result = Content. 1177 1178test(resumable_endpoint_option, [ 1179 setup((set_tus_options([tus_client_chunk_size(4000), 1180 tus_expiry_seconds(1) 1181 ]), 1182 random_file(tus_storage_test, Path), 1183 make_directory(Path), 1184 Base = 'http://cloudapi.com:8080/TerminusX/api/files', 1185 spawn_auth_server(URL, Port, [resumable_endpoint_base(Base),tus_storage_path(Path)]))), 1186 cleanup(kill_server(Port)) 1187 ]) :- 1188 1189 random_file('example_txt_tus', File), 1190 open(File, write, Stream), 1191 Content = "asdf fdsa yes yes yes", 1192 format(Stream, '~s', [Content]), 1193 close(Stream), 1194 tus_options(URL, Tus_Options, [authorization(basic(me,pass))]), 1195 tus_create(URL, File, _Length, Resource_URL, Tus_Options, [authorization(basic(me,pass))]), 1196 1197 string_length(Base, Len), 1198 sub_string(Resource_URL, 0, Len, _, Base). 1199 1200test(conflict, [ 1201 setup((set_tus_options([tus_client_chunk_size(4), 1202 tus_expiry_seconds(1) 1203 ]), 1204 random_file(tus_storage_test, Path), 1205 make_directory(Path), 1206 spawn_auth_server(URL, Port, [tus_storage_path(Path)]))), 1207 cleanup(kill_server(Port)), 1208 error(file_already_exists(File) ,_) 1209 ]) :- 1210 1211 random_file('example_txt_tus', File), 1212 open(File, write, Stream), 1213 Content = "And now for something completely different...", 1214 format(Stream, '~s', [Content]), 1215 close(Stream), 1216 1217 tus_upload(File, URL, _Resource1, [authorization(basic(me,pass))]), 1218 tus_upload(File, URL, _Resource2, [authorization(basic(me,pass))]). 1219 1220:- end_tests(tus).
TUS Protocol
Both client and server implementation of the TUS protocol.
The TUS protocol allows resumable file uploads via HTTP in swipl https://tus.io/
Server implementation
Requests are structured according to the prolog http library format.
Implemented features of the TUS protocol:
OPTIONS (Discovery) POST (Create)* HEAD (Find resumption point) PATCH (Send chunk)
Suggested
*/