1 /*
2  * Copyright (c) 2017-2018 sel-project
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to deal
6  * in the Software without restriction, including without limitation the rights
7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8  * copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in all
12  * copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20  * SOFTWARE.
21  *
22  */
23 module init;
24 
25 import std.algorithm : sort, canFind, clamp;
26 import std.array : join, split;
27 import std.ascii : newline;
28 import std.conv : ConvException, to;
29 import std.file;
30 import std.json;
31 import std.path : dirSeparator, buildNormalizedPath, absolutePath, relativePath;
32 import std.process : environment, executeShell;
33 import std.regex : matchFirst, ctRegex;
34 import std.stdio : writeln;
35 import std..string;
36 import std.zip;
37 
38 import selery.about;
39 
40 import toml;
41 import toml.json;
42 
43 enum Type {
44 
45 	default_ = "default",
46 	hub = "hub",
47 	node = "node"
48 
49 }
50 
51 int main(string[] args) {
52 
53 	string libraries;
54 	if(exists(".selery/libraries")) {
55 		// should be an absolute normalised path
56 		libraries = cast(string)read(".selery/libraries");
57 	} else {
58 		// assuming this file is executed in ../
59 		libraries = buildNormalizedPath(absolutePath(".."));
60 	}
61 	if(!libraries.endsWith(dirSeparator)) libraries ~= dirSeparator;
62 	
63 	bool portable = false;
64 	bool plugins = true;
65 	
66 	Type type = Type.default_;
67 	
68 	// generate files
69 	{
70 		// clear
71 		if(exists("views")) {
72 			try {
73 				foreach(file ; dirEntries("views", SpanMode.breadth)) {
74 					if(file.isFile) remove(file);
75 				}
76 			} catch(Exception) {}
77 		} else {
78 			mkdirRecurse("views");
79 		}
80 		write("views/version.txt", Software.displayVersion);
81 		string[] notes;
82 		string changelog = cast(string)read("../.github/changelog.md");
83 		immutable v = "### " ~ Software.displayVersion;
84 		auto start = changelog.indexOf(v);
85 		if(start != -1) {
86 			start += v.length;
87 			changelog = changelog[start..$];
88 			immutable end = changelog.indexOf("##");
89 			write("views/notes.txt", changelog[0..(end==-1?$:end)].strip.replace("\r", "").replace("\n", "\\n"));
90 		} else {
91 			write("views/notes.txt", "There are no release notes for this version.");
92 		}
93 		write("views/is_release.txt", to!string(environment.get("APPVEYOR_REPO_COMMIT_MESSAGE", "").indexOf("[release]") != -1));
94 		// ci info
95 		JSONValue[string] ci;
96 		if(environment.get("TRAVIS", "") == "true") {
97 			ci["name"] = "travis-ci";
98 			ci["repo"] = environment["TRAVIS_REPO_SLUG"];
99 			ci["job"] = environment["TRAVIS_JOB_NUMBER"];
100 		} else if(environment.get("APPVEYOR", "").toLower == "true") {
101 			ci["name"] = "appveyor";
102 			ci["repo"] = environment["APPVEYOR_REPO_NAME"];
103 			ci["job"] = environment["APPVEYOR_BUILD_NUMBER"] ~ "." ~ environment["APPVEYOR_JOB_NUMBER"];
104 		}
105 		if(ci.length) write("views/build_ci.json", JSONValue(ci).toString());
106 		// git info
107 		JSONValue[string] git;
108 		if(exists("../.git/")) {
109 			git["remote"] = executeShell("git config --get remote.origin.url").output.strip;
110 			git["branch"] = executeShell("git rev-parse --abbrev-ref HEAD").output.strip;
111 			git["head"] = executeShell("git rev-parse HEAD").output.strip;
112 		}
113 		write("views/build_git.json", JSONValue(git).toString());
114 	}
115 	
116 	foreach(arg ; args) {
117 		switch(arg.toLower()) {
118 			case "--no-plugins":
119 				plugins = false;
120 				break;
121 			case "--portable":
122 				portable = true;
123 				break;
124 			case "default":
125 			case "classic":
126 			case "allinone":
127 			case "all-in-one":
128 				type = Type.default_;
129 				break;
130 			case "hub":
131 				type = Type.hub;
132 				break;
133 			case "node":
134 				type = Type.node;
135 				break;
136 			default:
137 				break;
138 		}		
139 	}
140 	
141 	bool[string] active_plugins;
142 	
143 	if(exists("../build-plugins.toml")) {
144 		try {
145 			foreach(key, value; parseTOML(cast(string)read("../build-plugins.toml"))) {
146 				active_plugins[key] = value.type == TOML_TYPE.TRUE;
147 			}
148 		} catch(TOMLException) {}
149 	}
150 
151 	TOMLDocument[string] plugs; // plugs[location] = settingsfile
152 
153 	if(plugins) {
154 
155 		bool loadPlugin(string path) {
156 			if(!path.endsWith(dirSeparator)) path ~= dirSeparator;
157 			foreach(pack ; ["plugin.toml", "plugin.json"]) {
158 				if(exists(path ~ pack)) {
159 					if(pack.endsWith(".toml")) {
160 						auto toml = parseTOML(cast(string)read(path ~ pack));
161 						toml["single"] = false;
162 						plugs[path] = toml;
163 						return true;
164 					} else {
165 						auto json = parseJSON(cast(string)read(path ~ pack));
166 						if(json.type == JSON_TYPE.OBJECT) {
167 							json["single"] = false;
168 							plugs[path] = TOMLDocument(toTOML(json).table);
169 							return true;
170 						}
171 					}
172 				}
173 			}
174 			return false;
175 		}
176 		
177 		void loadZippedPlugin(string path) {
178 			// unzip and load as normal plugin
179 			auto data = read(path);
180 			auto zip = new ZipArchive(data);
181 			immutable name = path[path.lastIndexOf("/")+1..$-4];
182 			immutable dest = ".selery/plugins/" ~ name ~ "/";
183 			bool update = true;
184 			if(exists(dest)) {
185 				if(exists(dest ~ "crc32.json")) {
186 					update = false;
187 					auto json = parseJSON(cast(string)read(dest ~ "crc32.json")).object;
188 					// compare file names
189 					if(sort(json.keys).release() != sort(zip.directory.keys).release()) update = true;
190 					else {
191 						// compare file's crc32
192 						foreach(name, member; zip.directory) {
193 							if(member.crc32 != json[name].integer) {
194 								update = true;
195 								break;
196 							}
197 						}
198 					}
199 				}
200 				if(update) {
201 					foreach(string file ; dirEntries(dest, SpanMode.breadth)) {
202 						if(file.isFile) remove(file);
203 					}
204 				}
205 			} else {
206 				mkdirRecurse(dest);
207 			}
208 			if(update) {
209 				JSONValue[string] files;
210 				foreach(name, member; zip.directory) {
211 					files[name] = member.crc32;
212 					if(!name.endsWith("/")) {
213 						zip.expand(member);
214 						if(name.indexOf("/") != -1) mkdirRecurse(dest ~ name[0..name.lastIndexOf("/")]);
215 						write(dest ~ name, member.expandedData);
216 					}
217 				}
218 				write(dest ~ "crc32.json", JSONValue(files).toString());
219 			}
220 			if(!loadPlugin(dest)) loadPlugin(dest ~ name);
221 		}
222 
223 		void loadSinglePlugin(string location) {
224 			immutable name = location[location.lastIndexOf("/")+1..$-2].replace("-", "_");
225 			foreach(line ; split(cast(string)read(location), "\n")) {
226 				if(line.strip.startsWith("module") && line[6..$].strip.startsWith(name ~ ";")) {
227 					string main = name ~ ".";
228 					bool uppercase = true;
229 					foreach(c ; name) {
230 						if(c == '_') {
231 							uppercase = true;
232 						} else {
233 							if(uppercase) main ~= toUpper("" ~ c);
234 							else main ~= c;
235 							uppercase = false;
236 						}
237 					}
238 					plugs[location] = TOMLDocument(["name": TOMLValue(name.replace("_", "-")), "main": TOMLValue(main)]);
239 					break;
240 				}
241 			}
242 		}
243 
244 		writeln("Generating dub package for ", Software.name, " ", Software.displayVersion, ".");
245 
246 		// load plugins in plugins folder
247 		if(exists("../plugins")) {
248 			foreach(string ppath ; dirEntries("../plugins/", SpanMode.shallow)) {
249 				if(ppath.isDir) {
250 					loadPlugin(ppath);
251 				} else if(ppath.isFile && ppath.endsWith(".zip")) {
252 					loadZippedPlugin(ppath);
253 				} else if(ppath.isFile && ppath.endsWith(".d")) {
254 					loadSinglePlugin(ppath);
255 				}
256 			}
257 		}
258 
259 	}
260 
261 	Plugin[string] info;
262 	
263 	foreach(path, value; plugs) {
264 		Plugin plugin;
265 		plugin.name = value["name"].str;
266 		checkName(plugin.name);
267 		if(path.isFile) {
268 			plugin.single = buildNormalizedPath(absolutePath(path));
269 		}
270 		plugin.path = buildNormalizedPath(absolutePath(path));
271 		if(plugin.name !in info) {
272 			plugin.toml = value;
273 			if(!plugin.path.endsWith(dirSeparator)) plugin.path ~= dirSeparator;
274 			auto priority = "priority" in value;
275 			if(priority) {
276 				if(priority.type == TOML_TYPE.STRING) {
277 					immutable p = priority.str.toLower;
278 					plugin.priority = (p == "high" || p == "🔥") ? 10 : (p == "medium" || p == "normal" ? 5 : 1);
279 				} else if(priority.type == TOML_TYPE.INTEGER) {
280 					plugin.priority = clamp(priority.integer.to!size_t, 1, 10);
281 				}
282 			}
283 			auto authors = "authors" in value;
284 			auto author = "author" in value;
285 			if(authors && authors.type == TOML_TYPE.ARRAY) {
286 				foreach(a ; authors.array) {
287 					if(a.type == TOML_TYPE.STRING) {
288 						plugin.authors ~= a.str;
289 					}
290 				}
291 			} else if(author && author.type == TOML_TYPE.STRING) {
292 				plugin.authors = [author.str];
293 			}
294 			auto target = "target" in value;
295 			if(target && target.type == TOML_TYPE.STRING) {
296 				switch(target.str.toLower) {
297 					case "default":
298 						plugin.target = Type.default_;
299 						break;
300 					case "hub":
301 						plugin.target = Type.hub;
302 						break;
303 					case "node":
304 						plugin.target = Type.node;
305 						break;
306 					default:
307 						break;
308 				}
309 			}
310 			foreach(mname, mvalue; (plugin.target == Type.default_ ? ["hub-main": Type.hub, "node-main": Type.node] : ["main": plugin.target])) {
311 				auto mptr = mname in value;
312 				if(mptr && mptr.type == TOML_TYPE.STRING) {
313 					Main main;
314 					string[] spl = mptr.str.split(".");
315 					if(plugin.single.length) {
316 						main.module_ = spl[0];
317 						main.main = mptr.str;
318 					} else {
319 						immutable m = mptr.str.lastIndexOf(".");
320 						if(m != -1) {
321 							main.module_ = mptr.str[0..m];
322 							main.main = mptr.str;
323 						}
324 					}
325 					plugin.main[mvalue] = main;
326 				}
327 			}
328 			if(plugin.single.length) {
329 				plugin.version_ = "~single";
330 			} else {
331 				foreach(string file ; dirEntries(plugin.path ~ "src", SpanMode.breadth)) {
332 					if(file.isFile && file.endsWith(dirSeparator ~ "api.d")) {
333 						plugin.api = true;
334 						break;
335 					}
336 				}
337 				if(exists(plugin.path ~ ".git") && isDir(plugin.path ~ ".git")) {
338 					// try to get version using git
339 					immutable tag = executeShell("cd " ~ plugin.path ~ " && git describe --tags").output.strip; //TODO do not use &&
340 					if(tag.startsWith("v")) plugin.version_ = tag;
341 				}
342 			}
343 			info[plugin.name] = plugin;
344 		} else {
345 			throw new Exception("Plugin '" ~ plugin.name ~ "' at " ~ plugin.path ~ " conflicts with a plugin with the same name at " ~ info[plugin.name].path);
346 		}
347 	}
348 	
349 	// remove plugins disabled in plugins.toml
350 	foreach(plugin, enabled; active_plugins) {
351 		if(!enabled) {
352 			auto p = plugin in info;
353 			if(p) p.enabled = false;
354 		}
355 	}
356 
357 	auto ordered = info.values;
358 
359 	// sort by priority (or alphabetically)
360 	sort!"a.priority == b.priority ? a.name < b.name : a.priority > b.priority"(ordered);
361 
362 	// control api version
363 	foreach(ref inf ; ordered) {
364 		long[] api;
365 		auto ptr = "api" in inf.toml;
366 		if(ptr) {
367 			if((*ptr).type == TOML_TYPE.INTEGER) {
368 				api ~= (*ptr).integer;
369 			} else if((*ptr).type == TOML_TYPE.ARRAY) {
370 				foreach(v ; (*ptr).array) {
371 					if(v.type == TOML_TYPE.INTEGER) api ~= v.integer;
372 				}
373 			} else if((*ptr).type == TOML_TYPE.TABLE) {
374 				auto from = "from" in *ptr;
375 				auto to = "to" in *ptr;
376 				if(from && (*from).type == TOML_TYPE.INTEGER && to && (*to).type == TOML_TYPE.INTEGER) {
377 					foreach(a ; (*from).integer..(*to).integer+1) {
378 						api ~= a;
379 					}
380 				}
381 			}
382 		}
383 		if(api.length == 0 /*|| api.canFind(Software.api)*/) {
384 			writeln(inf.name, " ", inf.version_, ": loaded");
385 		} else {
386 			writeln(inf.name, " ", inf.version_, ": cannot load due to wrong api ", api);
387 			return 1;
388 		}
389 	}
390 	
391 	JSONValue[string] builder;
392 	builder["name"] = "selery-builder";
393 	builder["targetName"] = "selery" ~ (type!=Type.default_ ? "-" ~ type : "") ~ (portable ? "-" ~ Software.displayVersion : "");
394 	builder["targetType"] = "executable";
395 	builder["targetPath"] = "..";
396 	builder["workingDirectory"] = "..";
397 	builder["sourceFiles"] = ["main/" ~ type ~ ".d", ".selery/builder.d"];
398 	builder["dependencies"] = [
399 		"selery": ["path": ".."],
400 		"toml": ["version": "~>1.0.0-rc.3"],
401 		"toml:json": ["version": "~>1.0.0-rc.3"]
402 	];
403 	builder["configurations"] = [["name": cast(string)type]];
404 	builder["subPackages"] = new JSONValue[0];
405 		
406 	string loads = "";
407 	
408 	if(!exists(".selery")) mkdir(".selery");
409 	
410 	string[] pluginsFile;
411 	
412 	JSONValue[] json;
413 
414 	foreach(ref value ; ordered) {
415 		pluginsFile ~= value.name ~ " = " ~ value.enabled.to!string;
416 		if(value.enabled) {
417 			if(value.single.length) {
418 				builder["sourceFiles"].array ~= JSONValue(relativePath(value.single));
419 			} else {
420 				JSONValue[string] sub;
421 				sub["name"] = value.name;
422 				sub["targetType"] = "library";
423 				sub["targetPath"] = ".." ~ dirSeparator ~ "libs";
424 				sub["configurations"] = [["name": "plugin"]];
425 				sub["dependencies"] = ["selery": ["path": ".."]],
426 				sub["sourcePaths"] = [relativePath(value.path ~ "src")];
427 				sub["importPaths"] = [relativePath(value.path ~ "src")];
428 				auto dptr = "dependencies" in value.toml;
429 				if(dptr && dptr.type == TOML_TYPE.TABLE) {
430 					foreach(name, d; dptr.table) {
431 						if(name.startsWith("dub:")) {
432 							sub["dependencies"][name[4..$]] = toJSON(d);
433 						} else if(name == "dub" && d.type == TOML_TYPE.TABLE) {
434 							foreach(dname, dd; d.table) {
435 								sub["dependencies"][dname] = toJSON(dd);
436 							}
437 						} else {
438 							//TODO depends on another plugin
439 							sub["dependencies"][":" ~ name] = "*";
440 						}
441 					}
442 				}
443 				auto subConfigurations = "subConfigurations" in value.toml;
444 				if(subConfigurations && subConfigurations.type == TOML_TYPE.TABLE) {
445 					sub["subConfigurations"] = toJSON(*subConfigurations);
446 				}
447 				builder["subPackages"].array ~= JSONValue(sub);
448 				builder["dependencies"][":" ~ value.name] = "*";
449 			}
450 			foreach(string mname; value.target==Type.default_ ? [Type.hub, Type.node] : [value.target]) {
451 				auto main = mname in value.main;
452 				string load = "ret ~= new PluginOf!(" ~ (main ? main.main : "Object") ~ ")(`" ~ value.name ~ "`, `" ~ value.path ~ "`, " ~ value.authors.to!string ~ ", `" ~ value.version_ ~ "`);";
453 				auto conditions = "conditions" in value.toml;
454 				if(conditions && conditions.type == TOML_TYPE.TABLE) {
455 					string[] conds;
456 					foreach(key, value; conditions.table) {
457 						conds ~= "cond!(`" ~ key ~ "`, is_node)(config, " ~ to!string(value.type == TOML_TYPE.TRUE) ~ ")";
458 					}
459 					load = "if(" ~ conds.join("&&") ~ "){ " ~ load ~ " }";
460 				}
461 				if(value.single.length) load = "static if(is(" ~ value.main[Type.default_].main ~ " == class)){ " ~ load ~ " }";
462 				load = "static if(target == `" ~ mname ~ "`){ " ~ (main ? "static import " ~ main.module_ ~ "; " : "") ~ load ~ " }";
463 				loads ~= "\t" ~ load ~ "\n";
464 			}
465 			json ~= value.toJSON();
466 			if(portable) {
467 				// copy plugins/$plugin/assets into assets/plugins/$plugin
468 				immutable assets = value.path ~ "assets" ~ dirSeparator;
469 				if(exists(assets) && assets.isDir) {
470 					foreach(file ; dirEntries(assets, SpanMode.breadth)) {
471 						immutable dest = "../assets/plugins" ~ dirSeparator ~ value.name ~ dirSeparator ~ file[assets.length..$];
472 						if(file.isFile) {
473 							mkdirRecurse(dest[0..dest.lastIndexOf(dirSeparator)]);
474 							write(dest, read(file));
475 						}
476 					}
477 				}
478 			}
479 		}
480 	}
481 
482 	writeDiff(".selery/builder.d", "module pluginloader;\n\nimport selery.config : Config;\nimport selery.plugin : Plugin;\n\nimport condition;\n\nPlugin[] loadPlugins(alias PluginOf, string target)(inout Config config){\n\tPlugin[] ret;\n" ~ loads ~ "\treturn ret;\n}\n\nenum info = `" ~ JSONValue(json).toString() ~ "`;\n");
483 	
484 	writeDiff("dub.json", JSONValue(builder).toString());
485 	
486 	write("../build-plugins.toml", pluginsFile.join(newline) ~ newline);
487 
488 	if(portable) {
489 
490 		auto zip = new ZipArchive();
491 
492 		// get all files in assets
493 		foreach(string file ; dirEntries("../assets/", SpanMode.breadth)) {
494 			immutable name = file[10..$].replace("\\", "/");
495 			if(file.isFile && !name.startsWith(".") && !name.endsWith(".ico") && (!name.startsWith("web/") || name.endsWith("/main.css") || name.indexOf("/res/") != -1)) {
496 				//TODO optimise .lang files by removing empty lines, windows' line endings and comments
497 				auto data = read(file);
498 				auto member = new ArchiveMember();
499 				member.name = name;
500 				member.expandedData(cast(ubyte[])(file.endsWith(".json") ? parseJSON(cast(string)data).toString() : data));
501 				member.compressionMethod = CompressionMethod.deflate;
502 				zip.addMember(member);
503 			}
504 		}
505 		mkdirRecurse("views");
506 		write("views/portable.zip", zip.build());
507 
508 	} else if(exists("views/portable.zip")) {
509 
510 		remove("views/portable.zip");
511 
512 	}
513 	
514 	return 0;
515 
516 }
517 
518 enum invalid = ["selery", "sel", "toml", "default", "lite", "hub", "node", "builder", "condition", "config", "starter", "pluginloader"];
519 
520 void checkName(string name) {
521 	void error(string message) {
522 		throw new Exception("Cannot load plugin '" ~ name ~ "': " ~ message);
523 	}
524 	if(name.matchFirst(ctRegex!`[^a-z0-9\-]`)) error("Name contains characters outside the range a-z0-9-");
525 	if(name.length == 0 || name.length > 64) error("Invalid name length: " ~ name.length.to!string ~ " is not between 1 and 64");
526 	if(invalid.canFind(name)) error("Name is reserved");
527 }
528 
529 void writeDiff(string location, const void[] data) {
530 	if(!exists(location) || read(location) != data) write(location, data);
531 }
532 
533 struct Plugin {
534 
535 	bool enabled = true;
536 	bool api = false;
537 
538 	TOMLDocument toml;
539 	
540 	string single;
541 	
542 	size_t priority = 5;
543 	string path;
544 
545 	string name;
546 	string[] authors;
547 	string version_ = "~local";
548 	
549 	Type target = Type.default_;
550 	Main[Type] main;
551 	
552 	JSONValue toJSON() {
553 		JSONValue[string] ret;
554 		ret["path"] = path;
555 		ret["name"] = name;
556 		ret["authors"] = authors;
557 		ret["version"] = version_;
558 		ret["target"] = target;
559 		if(main.length) {
560 			JSONValue[string] mret;
561 			foreach(key, value; main) {
562 				mret[key] = value.toJSON();
563 			}
564 			ret["main"] = mret;
565 		}
566 		return JSONValue(ret);
567 	}
568 	
569 }
570 
571 struct Main {
572 
573 	string module_;
574 	string main;
575 	
576 	JSONValue toJSON() {
577 		return JSONValue([
578 			"module": module_,
579 			"main": main
580 		]);
581 	}
582 
583 }