Subversion Repositories yt_downloader

Rev

Rev 2 | Rev 6 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
2 daniel-mar 1
#!/usr/bin/php
2
<?php
3
 
5 daniel-mar 4
// ViaThinkSoft YouTube Downloader Util 2.1.1
5
// Revision: 2019-08-05
2 daniel-mar 6
// Author: Daniel Marschall <www.daniel-marschall.de>
5 daniel-mar 7
// Licensed under the terms of the Apache 2.0 license
2 daniel-mar 8
//
9
// Syntax:
10
// ./ytdwn [-t|--type v:[ext]|a:[ext]] (default v:)
11
//         [-o|--outputDir <dir>]      (default current working directory)
12
//         [-a|--alreadyDownloaded <file>]
13
//         [-f|--failList <file> <treshold>]  (This file logs failures)
14
//         [-F|--failTreshold <num>]   (Don't download if failure (-f) treshold is reached. Default: 3)
15
//         [-V|--version]              (shows version)
16
//         [-v|--verbose]              (displays verbose information to STDOUT)
17
//         [-h|--help]                 (shows help)
18
//         [-N|--no-mp3-tagtransfer]   (disables transfer of video ID to MP3 ID tag)
19
//                                     (This feature requires the package "id3v2")
20
//         [-T|--default-template <t>] (Sets default filename template.)
21
//                                     (Default: '%(title)s-%(id)s.%(ext)s')
22
//         [-X|--extra-args <args>]    (Additional arguments passed through)
23
//                                     (youtube-dl. Default "-ic")
24
//         [-A|--api-key <file|key>]   (specifies the API key, or a file containing the API key)
25
//                                     (Default: ~/.yt_api_key)
26
//         [-C|--resultcache <file>]   (allows video results to be cached in this file)
27
//                                     (only for playlists or channels)
28
//         [-O|--create-outputdir]     (allows creation of the output directories, recursively)
29
//         [--]
30
//         <resource> [<resource> ...]
31
//
32
// For all paths (outputDir, alreadyDownloaded, apikey, failList and resultcache), you can use the
33
// term '[listname]' which will be replaced by the basename of the current list file (without file extension).
34
// For example you can do following:
35
//         ./ytdwn -o 'downloads/[listname]' -- list:*.list
36
// If no list file is processed, it will be replaced with nothing.
37
//
38
// The "alreadyDownloaded" argument contains a file which will be managed by ytdwn.
39
// It will contain all video IDs which have been downloaded. This allows you to
40
// move away the already downloaded files, and ytdwn will not download them again.
41
//
42
// Examples for type:
43
//         v:         best video quality
44
//         a:         best audio only
45
//         a:mp3      audio only, mp3
46
//         Valid audio formats according to "man youtube-dl":
47
//         "best", "aac", "flac", "mp3", "m4a", "opus", "vorbis", or "wav"; "best" by default
48
//
49
// A <resource> can be one of the following:
50
//         vid:<video ID>
51
//         vurl:<youtube video URL>
52
//         pid:<playlist ID>
53
//         purl:<playlist URL>
54
//         cid:<channel id>
55
//         cname:<channel name>
56
//         curl:<channel or username URL>
57
//         list:<file with resource entries>   (comments can be #)
58
//         search:<searchterm>
59
//
60
// For channels (cid, cname, curl) you can also perform a search to filter the results.
61
// This can be done like this:
62
//         cname:[search="Elvis Presley"]channel_1234
63
// For the search option, following parameters are possible:
64
//         search:[order=date][maxresults=50]"Elvis Presley"
65
//         Acceptable order values are: date, rating, relevance, title, videoCount, viewCount
66
//         Default values are order=relevance and maxresults=10
67
//         Use maxresults=-1 to download everything which matches the searchterm.
68
//
69
// Requirements:
70
//         - PHP CLI
71
//         - Package "youtube-dl" (ytdwn will try to download it automatically, if possible)
72
//         - A YouTube API key (can be obtained here: https://console.developers.google.com/apis/credentials )
73
//         - If you want to extract audio, you need additionally: ffmpeg or avconv and ffprobe or avprobe.
74
//         - Optional: package "id3v2" to allow the YouTube video id to be transferred to the MP3 ID tag
75
 
76
// ------------------------------------------------------------------------------------------------
77
 
78
error_reporting(E_ALL | E_NOTICE | E_STRICT | E_DEPRECATED);
79
 
80
define('AUTO_API_KEY', '~/.yt_api_key');
81
define('DOWNLOAD_SIMULATION_MODE', false);
82
define('DEFAULT_SEARCH_ORDER', 'relevance');
83
define('DEFAULT_SEARCH_MAXRESULTS', 10);
84
 
85
putenv("LANG=de_DE.UTF-8"); // required if video titles contain non-ASCII symbols
86
 
87
require_once __DIR__ . '/youtube_functions.inc.phps';
88
 
89
// Check if we are running in command line
90
 
91
if (PHP_SAPI !== 'cli') {
92
	fwrite(STDERR, "Error: Can only run in CLI mode\n");
93
	exit(2);
94
}
95
 
96
// Global vars
97
 
98
$listFilenameStack = array();
99
 
100
// Default values
101
 
102
$allow_creation_outputdir = false;
103
$type = 'v:';
104
$outputDir = '';
105
$alreadyDownloaded = '';
106
$failList = '';
107
$failTreshold = 3;
108
$rest_args = array();
109
$verbose = false;
110
$mp3id_transfer = true;
111
$apikey = '';
112
$resultcache = '';
113
$extra_args =
114
//            '-k ' . // The additional "-k" option in the above makes youtube-dl keep downloaded videos.
115
              '-i ' . // continue upon download errors
116
              '-c ';  // resume partially downloaded video files
117
$default_template = '%(title)s-%(id)s.%(ext)s';
118
 
119
// Parse arguments
120
// We do not use getopt() at the moment, because the important functionality "optind" is only available in PHP 7.1, which is not yet distributed with most of the stable Linux distros
121
 
122
$init_extra_args = false;
123
$argv_bak = $argv;
124
array_shift($argv_bak);
125
while (count($argv_bak) > 0) {
126
	$arg = array_shift($argv_bak);
127
	$arg2 = $arg . ' ' . (isset($argv_bak[0]) ? $argv_bak[0] : '');
128
	if (preg_match('@^(/t|\-t|\-\-type)(\s+|=)(.*)$@s', $arg2, $m)) {
129
		array_shift($argv_bak);
130
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
131
		$type = $m[3];
132
	} else if (preg_match('@^(/o|\-o|\-\-outputDir)(\s+|=)(.*)$@s', $arg2, $m)) {
133
		array_shift($argv_bak);
134
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
135
		$outputDir = $m[3];
136
	} else if (preg_match('@^(/a|\-a|\-\-alreadyDownloaded)(\s+|=)(.*)$@s', $arg2, $m)) {
137
		array_shift($argv_bak);
138
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
139
		$alreadyDownloaded = $m[3];
140
	} else if (preg_match('@^(/f|\-f|\-\-failList)(\s+|=)(.*)$@s', $arg2, $m)) {
141
		array_shift($argv_bak);
142
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
143
		$failList = $m[3];
144
	} else if (preg_match('@^(/F|\-F|\-\-failTreshold)(\s+|=)(.*)$@s', $arg2, $m)) {
145
		array_shift($argv_bak);
146
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
147
		$failTreshold = $m[3];
148
	} else if (preg_match('@^(/C|\-C|\-\-resultcache)(\s+|=)(.*)$@s', $arg2, $m)) {
149
		array_shift($argv_bak);
150
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
151
		$resultcache = $m[3];
152
	} else if (preg_match('@^(/T|\-T|\-\-default-template)(\s+|=)(.*)$@s', $arg2, $m)) {
153
		array_shift($argv_bak);
154
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
155
		$default_template = $m[3];
156
	} else if (preg_match('@^(/A|\-A|\-\-api-key)(\s+|=)(.*)$@s', $arg2, $m)) {
157
		array_shift($argv_bak);
158
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
159
		$apikey = file_exists($m[3]) ? trim(file_get_contents($m[3])) : $m[3];
160
	} else if (preg_match('@^(/X|\-X|\-\-extra-args)(\s+|=)(.*)$@s', $arg2, $m)) {
161
		array_shift($argv_bak);
162
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
163
		if ($init_extra_args) {
164
			$extra_args .= ' ' . $m[3]; // user has multiple "-X" arguments. append.
165
		} else {
166
			$extra_args = $m[3]; // overwrite defaults
167
			$init_extra_args = true;
168
		}
169
	} else if (preg_match('@^(/\?|/h|\-\?|\-h|\-\-help)$@s', $arg, $m)) {
170
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
171
		help();
172
	} else if (preg_match('@^(/V|\-V|\-\-version)$@s', $arg, $m)) {
173
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
174
		version();
175
	} else if (preg_match('@^(/v|\-v|\-\-verbose)$@s', $arg, $m)) {
176
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
177
		$verbose = true;
178
	} else if (preg_match('@^(/N|\-N|\-\-no-mp3-tagtransfer)$@s', $arg, $m)) {
179
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
180
		$mp3id_transfer = false;
181
	} else if (preg_match('@^(/O|\-O|\-\-create-outputdir)$@s', $arg, $m)) {
182
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
183
		$allow_creation_outputdir = true;
184
	} else if ($arg == '--') {
185
		if (count($rest_args) > 0) syntax_error("Invalid argument: ".$rest_args[0]);
186
		$rest_args = $argv_bak;
187
		break;
188
	} else {
189
		$rest_args[] = $arg;
190
	}
191
}
192
unset($arg);
193
unset($argv_bak);
194
unset($init_extra_args);
195
 
196
// Validity checks
197
 
198
if ((substr($type,0,2) != 'a:') && (substr($type,0,2) != 'v:')) syntax_error("Type must be either 'v:' or 'a:'. '$type' is not valid.");
199
 
200
if (count($rest_args) == 0) syntax_error("Please enter at least one desired video for downloading");
201
 
202
if ($failTreshold <= 0) syntax_error("Fail treshold has invalid value. Must be >0.");
203
 
5 daniel-mar 204
// Try to download/update youtube-dl into local directory
2 daniel-mar 205
 
5 daniel-mar 206
$newest_version_md5 = get_latest_ytdl_md5sum();
207
if (!$newest_version_md5) {
208
	fwrite(STDERR, "Failed to get MD5 sum of latest version of 'youtube-dl' from GitHub. Will not try to download/update 'youtube-dl' into local directory.\n");
209
} else {
210
	if (!file_exists(__DIR__.'/youtube-dl') || ($newest_version_md5 != md5_file(__DIR__.'/youtube-dl'))) {
211
		// Try to download/update the file in our directory. It should be the newest available, since YT often breaks downloader
212
		if (file_exists(__DIR__.'/youtube-dl')) {
213
			echo "Trying to update 'youtube-dl' in local directory...\n";
214
		} else {
215
			echo "Trying to download 'youtube-dl' into local directory...\n";
216
		}
217
		if (!@file_put_contents(__DIR__.'/youtube-dl', file_get_contents('https://yt-dl.org/downloads/latest/youtube-dl'))) {
218
			fwrite(STDERR, "Failed to download 'youtube-dl' into local directory (file_get_contents).\n");
219
		} else {
220
			if (!@chmod(__DIR__.'/youtube-dl', 0544)) {
221
				fwrite(STDERR, "Failed to download 'youtube-dl' into local directory (chmod 544).\n");
222
				@unlink(__DIR__.'/youtube-dl'); // try to delete, otherwise we might try to execute a non-executable file
223
			}
224
		}
225
	}
226
}
227
 
2 daniel-mar 228
if (command_exists(__DIR__.'/youtube-dl')) {
5 daniel-mar 229
	echo "Will use 'youtube-dl' from local directory\n";
2 daniel-mar 230
	define('YTDL_EXE', __DIR__.'/youtube-dl');
231
} else {
5 daniel-mar 232
	// Download failed. Is at least a package installed?
233
	if (command_exists('youtube-dl')) {
234
		echo "Will use 'youtube-dl' from Linux package\n";
235
		define('YTDL_EXE', 'youtube-dl');
2 daniel-mar 236
	} else {
237
		fwrite(STDERR, "This script requires the tool/package 'youtube-dl'. Please install it first.\n");
238
		exit(1);
239
	}
240
}
241
 
242
// Now process the videos
243
 
244
yt_set_apikey_callback('_getApikey');
245
 
246
foreach ($rest_args as $resource) {
247
	if ($verbose) echo "Handle: $resource\n";
248
	if (strpos($resource, ':') === false) {
249
		fwrite(STDERR, "Invalid resource '$resource' (you are missing the prefix, e.g. vurl: or vid:). Skipping.\n");
250
	} else {
251
		list($resourceType, $resourceValue) = explode(':', $resource, 2);
252
		ytdwn_handle_resource($resourceType, $resourceValue);
253
	}
254
}
255
 
256
// ------------------------------------------------------------------------------------------------
257
 
258
function ytdwn_handle_resource($resourceType, $resourceValue) {
259
	if ($resourceType == 'vid') {
260
		$video_id = parse_quoting($resourceValue);
261
		ytdwn_video_id($video_id);
262
	} else if ($resourceType == 'vurl') {
263
		$video_url = parse_quoting($resourceValue);
264
		$video_id  = getVideoIDFromURL($video_url);
265
		if (!$video_id) {
266
			fwrite(STDERR, "$video_url is not a valid YouTube video URL. Skipping.\n");
267
		} else {
268
			ytdwn_video_id($video_id);
269
		}
270
	} else if ($resourceType == 'pid') {
271
		$playlist_id = parse_quoting($resourceValue);
272
		ytdwn_playlist_id($playlist_id);
273
	} else if ($resourceType == 'purl') {
274
		$playlist_url = parse_quoting($resourceValue);
275
		$playlist_id  = getPlaylistIDFromURL($playlist_url);
276
		if (!$playlist_id) {
277
			fwrite(STDERR, "$playlist_url is not a valid YouTube playlist URL. Skipping\n");
278
		} else {
279
			ytdwn_playlist_id($playlist_id);
280
		}
281
	} else if ($resourceType == 'cid') {
282
		$channel_id = parse_quoting($resourceValue);
283
 
284
		if (preg_match('@\[search=(.+)\]@ismU', $channel_id, $m)) {
285
			$search = $m[1];
286
			$channel_id = preg_replace('@\[search=(.+)\]@ismU', '', $channel_id);
287
		} else {
288
			$search = ''; // default
289
		}
290
		$search = parse_quoting($search);
291
 
292
		ytdwn_channel_id($channel_id, $search);
293
	} else if ($resourceType == 'cname') {
294
		$channel_name = parse_quoting($resourceValue);
295
 
296
		if (preg_match('@\[search=(.+)\]@ismU', $channel_name, $m)) {
297
			$search = $m[1];
298
			$channel_name = preg_replace('@\[search=(.+)\]@ismU', '', $channel_name);
299
		} else {
300
			$search = ''; // default
301
		}
302
		$search = parse_quoting($search);
303
 
304
		$channel_name = parse_quoting($channel_name);
305
		$channel_id = yt_get_channel_id($channel_name);
306
		if (!$channel_id) {
307
			fwrite(STDERR, "URL $channel_name is a valid YouTube channel or username. Skipping.\n");
308
		} else {
309
			ytdwn_channel_id($channel_id, $search);
310
		}
311
	} else if ($resourceType == 'curl') {
312
		$channel_url = parse_quoting($resourceValue);
313
 
314
		if (preg_match('@\[search=(.+)\]@ismU', $channel_url, $m)) {
315
			$search = $m[1];
316
			$channel_url = preg_replace('@\[search=(.+)\]@ismU', '', $channel_url);
317
		} else {
318
			$search = ''; // default
319
		}
320
		$search = parse_quoting($search);
321
 
322
		$channel_url = parse_quoting($channel_url);
323
		$channel_id = curl_to_cid($channel_url);
324
		if (!$channel_id) {
325
			fwrite(STDERR, "URL $channel_url is a valid YouTube channel oder username URL. Skipping\n");
326
		} else {
327
			ytdwn_channel_id($channel_id, $search);
328
		}
329
	} else if ($resourceType == 'search') {
330
		$searchterm = parse_quoting($resourceValue);
331
 
332
		$order = '';
333
		if (preg_match('@\[order=(.+)\]@ismU', $searchterm, $m)) {
334
			$order = $m[1];
335
			$searchterm = preg_replace('@\[order=(.+)\]@ismU', '', $searchterm);
336
		} else {
337
			$order = DEFAULT_SEARCH_ORDER; // default
338
		}
339
		$order = parse_quoting($order);
340
 
341
		$maxresults = '';
342
		if (preg_match('@\[maxresults=(.+)\]@ismU', $searchterm, $m)) {
343
			$maxresults = $m[1];
344
			$searchterm = preg_replace('@\[maxresults=(.+)\]@ismU', '', $searchterm);
345
		} else {
346
			$maxresults = DEFAULT_SEARCH_MAXRESULTS; // default
347
		}
348
		$maxresults = parse_quoting($maxresults);
349
 
350
		$searchterm = parse_quoting($searchterm);
351
 
352
		ytdwn_search($searchterm, $order, $maxresults);
353
	} else if ($resourceType == 'list') {
354
		$list_files = glob(parse_quoting($resourceValue)); // in case the user entered a wildcard, e.g. *.list
355
		foreach ($list_files as $list_file) {
356
			if (!file_exists($list_file)) {
357
				fwrite(STDERR, "List file $list_file does not exist. Skipping\n");
358
			} else {
359
				ytdwn_list_file($list_file);
360
			}
361
		}
362
	} else {
363
		fwrite(STDERR, "Resource type '$resourceType' is not valid. Skipping $resourceType:$resourceValue.\n");
364
	}
365
}
366
 
367
function ytdwn_list_file($list_file) {
368
	global $listFilenameStack, $verbose;
369
 
370
	if ($verbose) echo "Processing list file '$list_file' ...\n";
371
 
372
	$listFilenameStack[] = $list_file;
373
	$lines = file($list_file);
374
	foreach ($lines as $line) {
375
		$line = trim($line);
376
		if ($line == '') continue;
377
		if ($line[0] == '#') continue;
378
		if (strpos($line, ':') === false) {
379
			fwrite(STDERR, "Invalid resource '$line' (you are missing the prefix, e.g. vurl: or vid:). Skipping.\n");
380
		} else {
381
			list($resourceType, $resourceValue) = explode(':',$line,2);
382
			ytdwn_handle_resource($resourceType, $resourceValue);
383
		}
384
	}
385
	array_pop($listFilenameStack);
386
}
387
 
388
function ytdwn_channel_id($channel_id, $search='') {
389
	global $type;
390
	global $verbose;
391
 
392
	if ($verbose) echo "Processing channel ID '$channel_id' ...\n";
393
 
394
	// List the videos of the channel
395
 
396
	$cont = !empty(_getResultcache()) && file_exists(_getResultcache()) ? file_get_contents(_getResultcache()) : '';
397
	$out = json_decode($cont, true);
398
	if ($out == NULL) $out = array();
399
 
400
	if (!empty(_getResultcache())) {
401
		$stats = yt_get_channel_stats($channel_id);
402
		$videocount = $stats['videoCount'];
403
 
404
		$key = (!empty($search)) ? 'cid:'.$channel_id.'/'.$search : 'cid:'.$channel_id;
405
 
406
		if (!isset($out[$key])) $out[$key] = array();
407
		$videocount_old = isset($out[$key]['count']) ? $out[$key]['count'] : -1;
408
	} else {
409
		$videocount = -1;
410
		$videocount_old = -2;
411
		$key = '';
412
	}
413
 
414
	if ($videocount_old != $videocount) { // Attention: This might not work if videos are deleted and added (= the count stays the same)
415
		if ($verbose && !empty(_getResultcache())) echo "Video count changed from $videocount_old to $videocount\n";
416
		$out[$key]['count'] = $videocount;
417
		if (!empty($search)) {
418
			$out[$key]['results'] = yt_channel_items($channel_id, $search);
419
		} else {
420
			$out[$key]['results'] = yt_channel_items($channel_id);
421
		}
422
	} else {
423
		if ($verbose) echo "Video count for channel is still $videocount, keep ".count($out[$key]['results'])." results.\n";
424
	}
425
 
426
	// Save the cache
427
 
428
	try {
429
		if (!empty(_getResultcache())) file_put_contents(_getResultcache(), json_encode($out));
430
	} catch(Exception $e) {
431
		fwrite(STDERR, "Cannot write result cache\n");
432
	}
433
 
434
	// Now download
435
 
436
	foreach ($out[$key]['results'] as list($id, $title)) {
437
		if ($verbose) echo "Downloading '$title' as ".hf_type($type)." ...\n";
438
		ytdwn_video_id($id);
439
	}
440
}
441
 
442
function ytdwn_playlist_id($playlist_id) {
443
	global $type;
444
	global $verbose;
445
 
446
	if ($verbose) echo "Processing playlist ID '$playlist_id' ...\n";
447
 
448
	// List the videos of the playlist
449
 
450
	$cont = !empty(_getResultcache()) && file_exists(_getResultcache()) ? file_get_contents(_getResultcache()) : '';
451
	$out = json_decode($cont, true);
452
	if ($out == NULL) $out = array();
453
 
454
	if (!empty(_getResultcache())) {
455
		$stats = yt_get_playlist_stats($playlist_id);
456
		$videocount = $stats['itemCount'];
457
 
458
		$key = 'pid:'.$playlist_id;
459
 
460
		if (!isset($out[$key])) $out[$key] = array();
461
		$videocount_old = isset($out[$key]['count']) ? $out[$key]['count'] : -1;
462
	} else {
463
		$videocount = -1;
464
		$videocount_old = -2;
465
		$key = '';
466
	}
467
 
468
	if ($videocount_old != $videocount) { // Attention: This might not work if videos are deleted and added (= the count stays the same)
469
		if ($verbose && !empty(_getResultcache())) echo "Video count changed from $videocount_old to $videocount\n";
470
		$out[$key]['count'] = $videocount;
471
		$out[$key]['results'] = yt_playlist_items($playlist_id);
472
	} else {
473
		if ($verbose) echo "Video count for playlist is still $videocount, keep ".count($out[$key]['results'])." results.\n";
474
	}
475
 
476
	// Save the cache
477
 
478
	try {
479
		if (!empty(_getResultcache())) file_put_contents(_getResultcache(), json_encode($out));
480
	} catch(Exception $e) {
481
		fwrite(STDERR, "Cannot write result cache\n");
482
	}
483
 
484
	// Now download
485
 
486
	foreach ($out[$key]['results'] as list($id, $title)) {
487
		if ($verbose) echo "Downloading '$title' as ".hf_type($type)." ...\n";
488
		ytdwn_video_id($id);
489
	}
490
}
491
 
492
function ytdwn_search($search, $order='', $maxresults=-1) {
493
	global $type;
494
	global $verbose;
495
 
496
	if ($verbose) echo "Searching for '$search' ...\n";
497
 
498
	// Perform the search and list the videos
499
 
500
	$results = yt_search_items($search, $order, $maxresults);
501
 
502
	// Now download
503
 
504
	foreach ($results as list($id, $title)) {
505
		if ($verbose) echo "Downloading '$title' as ".hf_type($type)." ...\n";
506
		ytdwn_video_id($id);
507
	}
508
}
509
 
510
function ytdwn_video_id($video_id) {
511
	global $type;
512
	global $verbose;
513
	global $mp3id_transfer;
514
	global $extra_args;
515
	global $default_template;
516
	global $failTreshold;
517
 
518
	if (DOWNLOAD_SIMULATION_MODE) {
519
		echo "SIMULATE download of video id $video_id as ".hf_type($type)." to "._getOutputDir()."\n";
520
		return;
521
	}
522
 
523
	if (!empty(_getAlreadyDownloaded()) && in_alreadydownloaded_file($type, $video_id)) {
524
		if ($verbose) echo "Video $video_id has already been downloaded. Skip.\n";
525
		return true;
526
	}
527
 
528
	if (!empty(_getFailList()) && (ytdwn_fail_counter($type, $video_id) >= $failTreshold)) {
529
		if ($verbose) echo "Video $video_id has failed too often. Skip.\n";
530
		return true;
531
	}
532
 
533
	$out = '';
534
	$code = -1;
535
 
536
	$outputTemplate = rtrim(_getOutputDir(), '/').'/'.$default_template;
537
 
538
	if (substr($type,0,2) == 'v:') {
539
		$format = substr($type,2);
540
		if (!empty($format)) {
541
			exec(YTDL_EXE.' -o '.escapeshellarg($outputTemplate).' '.$extra_args.' '.escapeshellarg(vid_to_vurl($video_id)).' --format '.escapeshellarg($format), $out, $code);
542
		} else {
543
			exec(YTDL_EXE.' -o '.escapeshellarg($outputTemplate).' '.$extra_args.' '.escapeshellarg(vid_to_vurl($video_id)), $out, $code);
544
		}
545
	} else if (substr($type,0,2) == 'a:') {
546
		$format = substr($type,2);
547
		if (!empty($format)) {
548
			exec(YTDL_EXE.' -o '.escapeshellarg($outputTemplate).' '.$extra_args.' '.escapeshellarg(vid_to_vurl($video_id)).' --extract-audio --audio-format '.escapeshellarg($format), $out, $code);
549
		} else {
550
			exec(YTDL_EXE.' -o '.escapeshellarg($outputTemplate).' '.$extra_args.' '.escapeshellarg(vid_to_vurl($video_id)).' --extract-audio', $out, $code);
551
		}
552
		if (($mp3id_transfer) && ($format == 'mp3')) mp3_transfer_vid_to_id();
553
	} else assert(false);
554
 
555
	if ($code == 0) {
556
		if ($verbose) fwrite(STDOUT, "Successfully downloaded video ID $video_id as ".hf_type($type)."\n");
557
		if (!empty(_getAlreadyDownloaded())) {
558
			try {
559
				addto_alreadydownloaded_file($type, $video_id);
560
			} catch(Exception $e) {
561
				fwrite(STDERR, "Cannot add to already downloaded file\n");
562
			}
563
		}
564
	} else {
565
		fwrite(STDERR, "Error downloading $video_id! (Code $code)\n");
566
		if (!empty(_getFailList())) {
567
			try {
568
				ytdwn_register_fail($type, $video_id, $code);
569
			} catch(Exception $e) {
570
				fwrite(STDERR, "Cannot register fail\n");
571
			}
572
		}
573
		return false;
574
	}
575
 
576
	return true;
577
}
578
 
579
function vid_to_vurl($video_id) {
580
	return "http://www.youtube.com/watch/?v=$video_id";
581
}
582
 
583
function EndsWith($Haystack, $Needle){
584
	return strrpos($Haystack, $Needle) === strlen($Haystack)-strlen($Needle);
585
}
586
 
587
function mp3_transfer_vid_to_id() {
588
	global $verbose;
589
	global $default_template;
590
 
591
	if (!command_exists('id3v2')) {
592
		if ($verbose) echo "Note: Tool id3v2 is not installed. Will not transfer the YouTube ID into the MP3 ID Tag\n";
593
		return false;
594
	}
595
 
596
	if (!EndsWith($default_template, '-%(id)s.%(ext)s'))  {
597
		if ($verbose) echo "Note: Cannot transfer video tag to MP3 because default template does not end with '-%(id)s.%(ext)s'.\n";
598
		return false;
599
	}
600
 
601
	$allok = true;
602
	$files = glob(rtrim(_getOutputDir(), '/').'/*-???????????.mp3');
603
	foreach ($files as $filename) {
604
		if (!preg_match('@-([a-zA-Z0-9\-_]{11})\.mp3$@ismU', $filename, $m)) continue;
605
		$yt_id = $m[1];
606
 
607
		if (!yt_check_video_id($yt_id)) continue; // just to be sure
608
 
609
		$orig_ts = filemtime($filename);
610
		system('id3v2 -c '.escapeshellarg($yt_id).' '.escapeshellarg($filename), $ec);
611
		touch($filename, $orig_ts);
612
		if ($ec != 0) {
613
			fwrite(STDERR, "Cannot set ID tag for file $filename\n");
614
			$allok = false;
615
			continue;
616
		}
617
 
618
		$target_filename = str_replace("-$yt_id.mp3", '.mp3', $filename);
619
		if (!intelligent_rename($filename, $target_filename)) {
620
			fwrite(STDERR, "Cannot move file $filename to $target_filename\n");
621
			$allok = false;
622
			continue;
623
		}
624
	}
625
	return $allok;
626
}
627
 
628
function curl_to_cid($channel_url) {
629
	if (preg_match("@https{0,1}://(www\\.|)youtube\\.com/user/(.*)(&|\\?)@ismU", $channel_url.'&', $m)) {
630
		$username = $m[2];
631
		$channel_id = yt_get_channel_id($username);
632
		return $channel_id;
633
	} else if (preg_match("@https{0,1}://(www\\.|)youtube\\.com/channel/(.*)(&|\\?)@ismU", $channel_url.'&', $m)) {
634
		$channel_id = $m[2];
635
		return $channel_id;
636
	} else {
637
		return false;
638
	}
639
}
640
 
641
function in_alreadydownloaded_file($type, $video_id) {
642
	$lines = file(_getAlreadyDownloaded());
643
	foreach ($lines as $line) {
644
		if (trim($line) == rtrim($type,':').':'.$video_id) {
645
			return true;
646
		}
647
	}
648
	return false;
649
}
650
 
651
function addto_alreadydownloaded_file($type, $video_id) {
652
	file_put_contents(_getAlreadyDownloaded(), rtrim($type,':').':'.$video_id."\n", FILE_APPEND);
653
}
654
 
655
function syntax_error($msg) {
656
	fwrite(STDERR, "Syntax error: ".trim($msg)."\n");
657
	fwrite(STDERR, "Please use argument '--help' to show the syntax rules.\n");
658
	exit(2);
659
}
660
 
661
function _help() {
662
	global $argv;
663
	$out = '';
664
	$own = file_get_contents($argv[0]);
665
	$help = explode('// ----', $own, 2)[0];
666
	$help = preg_match_all('@^//(.*)$@mU', $help, $m);
667
	foreach ($m[1] as $line) {
668
		$out .= substr($line,1)."\n";
669
	}
670
	return $out;
671
}
672
 
673
function help() {
674
	echo _help();
675
	exit(0);
676
}
677
 
678
function version() {
679
	echo explode("\n\n", _help(), 2)[0]."\n";
680
	exit(0);
681
}
682
 
683
function command_exists($command) {
684
	// https://stackoverflow.com/questions/592620/check-if-a-program-exists-from-a-bash-script
685
 
686
	$ec = -1;
687
	system('command -v '.escapeshellarg($command).' > /dev/null', $ec);
688
	return ($ec == 0);
689
}
690
 
691
function hf_type($type) {
692
	if (strpos($type, ':') === false) return $type; // invalid type (missing ':')
693
	list($av, $format) = explode(':', $type);
694
 
695
	if ($av == 'a') $av = 'audio';
696
	else if ($av == 'v') $av = 'video';
697
	else return $type; // invalid type
698
 
699
	return (!empty($format)) ? $format.'-'.$av : $av;
700
}
701
 
702
function expand_tilde($path) {
703
	// Source: http://jonathonhill.net/2013-09-03/tilde-expansion-in-php/
704
 
705
	if (function_exists('posix_getuid') && strpos($path, '~') !== false) {
706
		$info = posix_getpwuid(posix_getuid());
707
		$path = str_replace('~', $info['dir'], $path);
708
	}
709
 
710
	return $path;
711
}
712
 
713
function _getLastListname() {
714
	global $listFilenameStack;
715
	$listname = ''; // default
716
	if (count($listFilenameStack) > 0) {
717
		$listname = $listFilenameStack[count($listFilenameStack)-1];
718
		$listname = pathinfo($listname, PATHINFO_FILENAME); // remove file extension, e.g. ".list"
719
	}
720
	return $listname;
721
}
722
 
723
function _getApiKey() {
724
	global $apikey;
725
 
726
	$out = $apikey;
727
	if (empty($out)) {
728
		$auto_api_key = AUTO_API_KEY;
729
		$auto_api_key = expand_tilde($auto_api_key);
730
		$auto_api_key = str_replace('[listname]', _getLastListname(), $auto_api_key);
731
 
732
		if (file_exists($auto_api_key)) {
733
			$out = trim(file_get_contents($auto_api_key));
734
		} else {
735
			syntax_error("Please specify a YouTube API key with argument '-A'.");
736
		}
737
	} else {
738
		$out = str_replace('[listname]', _getLastListname(), $out);
739
		$out = expand_tilde($out);
740
 
741
		if (file_exists($out)) {
742
			$out = trim(file_get_contents($out));
743
		} else {
744
			// Assume, $out is a key, not a file
745
		}
746
	}
747
 
748
	if (!yt_check_apikey_syntax($out)) syntax_error("'$out' is not a valid API key, not an existing file containing an API key.\n");
749
 
750
	return $out;
751
}
752
 
753
function _getResultCache() {
754
	global $resultcache;
755
	if (empty($resultcache)) return '';
756
 
757
	$out = expand_tilde($resultcache);
758
 
759
	$out = str_replace('[listname]', _getLastListname(), $out);
760
 
761
	if (!file_exists($out)) {
762
		@touch($out);
763
		if (!file_exists($out)) {
764
			fwrite(STDERR, "File '$out' cannot be created. Disable feature.\n");
765
			return '';
766
		}
767
	}
768
 
769
	return $out;
770
}
771
 
772
function _getAlreadyDownloaded() {
773
	global $alreadyDownloaded;
774
	if (empty($alreadyDownloaded)) return '';
775
 
776
	$out = expand_tilde($alreadyDownloaded);
777
 
778
	$out = str_replace('[listname]', _getLastListname(), $out);
779
 
780
	if (!file_exists($out)) {
781
		@touch($out);
782
		if (!file_exists($out)) {
783
			fwrite(STDERR, "File '$out' cannot be created. Disable feature.\n");
784
			return '';
785
		}
786
	}
787
 
788
	return $out;
789
}
790
 
791
function _getFailList() {
792
	global $failList;
793
	if (empty($failList)) return '';
794
 
795
	$out = expand_tilde($failList);
796
 
797
	$out = str_replace('[listname]', _getLastListname(), $out);
798
 
799
	if (!file_exists($out)) {
800
		@touch($out);
801
		if (!file_exists($out)) {
802
			fwrite(STDERR, "File '$out' cannot be created. Disable feature.\n");
803
			return '';
804
		}
805
	}
806
 
807
	return $out;
808
}
809
 
810
function _getOutputDir() {
811
	global $outputDir, $allow_creation_outputdir;
812
	if (empty($outputDir)) return '.';
813
 
814
	$out = expand_tilde($outputDir);
815
 
816
	$out = str_replace('[listname]', _getLastListname(), $out);
817
 
818
	if ($allow_creation_outputdir) {
819
		if (!is_dir($out)) {
820
			mkdir($out, true);
821
			if (!is_dir($out)) {
822
				fwrite(STDERR, "Output directory '$out' does not exist.\n");
823
				exit(1);
824
			}
825
		}
826
	} else {
827
		if (!is_dir($out)) {
828
			fwrite(STDERR, "Output directory '$out' does not exist.\n");
829
			exit(1);
830
		}
831
	}
832
 
833
	return $out;
834
}
835
 
836
function parse_quoting($str) {
837
	if ((substr($str,0,1) == '"') && (substr($str,-1,1) == '"')) {
838
		$str = substr($str,1,strlen($str)-2);
839
 
840
		$escape = false;
841
		$out = '';
842
		for ($i=0; $i<strlen($str); $i++) {
843
			$char = $str[$i];
844
 
845
			if ($char == '\\') {
846
				if ($escape) {
847
					$out .= $char;
848
					$escape = false;
849
				} else {
850
					$escape = true;
851
				}
852
			} else {
853
				$out .= $char;
854
			}
855
 
856
		}
857
		$str = $out;
858
 
859
	}
860
 
861
	return $str;
862
}
863
 
864
function ytdwn_register_fail($type, $video_id, $code) {
865
	// Note: Error code $code ist currently not used
866
 
867
	$file = _getFailList();
868
	$cont = file_get_contents($file);
869
	if (preg_match("@Video ID ".preg_quote($video_id,'@')." failed (\d+) time\(s\) with type ".preg_quote($type,'@')."@ismU", $cont, $m)) {
870
		$cont = preg_replace("@Video ID ".preg_quote($video_id,'@')." failed (\d+) time\(s\) with type ".preg_quote($type,'@')."@ismU",
871
		                     "Video ID $video_id failed ".($m[1]+1)." time(s) with type $type", $cont);
872
		file_put_contents($file, $cont);
873
	} else {
874
		file_put_contents($file, "Video ID $video_id failed 1 time(s) with type $type\n", FILE_APPEND);
875
	}
876
}
877
 
878
function ytdwn_fail_counter($type, $video_id) {
879
	$file = _getFailList();
880
	$cont = file_get_contents($file);
881
	if (preg_match("@Video ID ".preg_quote($video_id,'@')." failed (\d+) time\(s\) with type ".preg_quote($type,'@')."@ismU", $cont, $m)) {
882
		return $m[1];
883
	} else {
884
		return 0;
885
	}
886
}
887
 
888
function intelligent_rename($src, $dest) {
889
	$pos = strrpos($dest, '.');
890
	$ext = substr($dest, $pos);
891
	$namewoext = substr($dest, 0, $pos);
892
	$failcnt = 1;
893
	$dest_neu = $dest;
894
	while (file_exists($dest_neu)) {
895
		$failcnt++;
896
		$dest_neu = "$namewoext ($failcnt)$ext";
897
	}
898
	return rename($src, $dest_neu);
899
}
5 daniel-mar 900
 
901
function get_latest_ytdl_md5sum() {
902
	$ch = curl_init();
903
	curl_setopt($ch, CURLOPT_URL, 'https://yt-dl.org/downloads/latest/MD5SUMS');
904
	#curl_setopt($ch, CURLOPT_HEADER, false);
905
	curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
906
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
907
	$cont = curl_exec($ch);
908
	if (preg_match('@^(.+)  youtube\-dl$@ismU', $cont, $m)) {
909
		return $m[1];
910
	} else {
911
		return false;
912
	}
913
}