1 /* 2 * Copyright (c) 2017-2019 sel-project 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a copy 5 * of this software and associated documentation files (the "Software"), to deal 6 * in the Software without restriction, including without limitation the rights 7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 * copies of the Software, and to permit persons to whom the Software is 9 * furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in all 12 * copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 * SOFTWARE. 21 * 22 */ 23 module init; 24 25 import std.algorithm : sort, canFind, clamp; 26 import std.array : join, split; 27 import std.ascii : newline; 28 import std.conv : ConvException, to; 29 import std.file; 30 import std.json; 31 import std.path : dirSeparator, buildNormalizedPath, absolutePath, relativePath; 32 import std.process : environment, executeShell; 33 import std.regex : matchFirst, ctRegex; 34 import std.stdio : writeln; 35 import std..string; 36 import std.zip; 37 38 import selery.about; 39 40 import toml; 41 import toml.json; 42 43 enum Type { 44 45 default_ = "default", 46 hub = "hub", 47 node = "node" 48 49 } 50 51 int main(string[] args) { 52 53 string libraries; 54 if(exists(".selery/libraries")) { 55 // should be an absolute normalised path 56 libraries = cast(string)read(".selery/libraries"); 57 } else { 58 // assuming this file is executed in ../ 59 libraries = buildNormalizedPath(absolutePath("..")); 60 } 61 if(!libraries.endsWith(dirSeparator)) libraries ~= dirSeparator; 62 63 bool portable = false; 64 bool plugins = true; 65 66 Type type = Type.default_; 67 68 // generate files 69 { 70 // clear 71 if(exists("views")) { 72 try { 73 foreach(file ; dirEntries("views", SpanMode.breadth)) { 74 if(file.isFile) remove(file); 75 } 76 } catch(Exception) {} 77 } else { 78 mkdirRecurse("views"); 79 } 80 write("views/version.txt", Software.displayVersion); 81 string[] notes; 82 string changelog = cast(string)read("../.github/changelog.md"); 83 immutable v = "### " ~ Software.displayVersion; 84 auto start = changelog.indexOf(v); 85 if(start != -1) { 86 start += v.length; 87 changelog = changelog[start..$]; 88 immutable end = changelog.indexOf("##"); 89 write("views/notes.txt", changelog[0..(end==-1?$:end)].strip.replace("\r", "").replace("\n", "\\n")); 90 } else { 91 write("views/notes.txt", "There are no release notes for this version."); 92 } 93 write("views/is_release.txt", to!string(environment.get("APPVEYOR_REPO_COMMIT_MESSAGE", "").indexOf("[release]") != -1)); 94 // ci info 95 JSONValue[string] ci; 96 if(environment.get("TRAVIS", "") == "true") { 97 ci["name"] = "travis-ci"; 98 ci["repo"] = environment["TRAVIS_REPO_SLUG"]; 99 ci["job"] = environment["TRAVIS_JOB_NUMBER"]; 100 } else if(environment.get("APPVEYOR", "").toLower == "true") { 101 ci["name"] = "appveyor"; 102 ci["repo"] = environment["APPVEYOR_REPO_NAME"]; 103 ci["job"] = environment["APPVEYOR_BUILD_NUMBER"] ~ "." ~ environment["APPVEYOR_JOB_NUMBER"]; 104 } 105 if(ci.length) write("views/build_ci.json", JSONValue(ci).toString()); 106 // git info 107 JSONValue[string] git; 108 if(exists("../.git/")) { 109 git["remote"] = executeShell("git config --get remote.origin.url").output.strip; 110 git["branch"] = executeShell("git rev-parse --abbrev-ref HEAD").output.strip; 111 git["head"] = executeShell("git rev-parse HEAD").output.strip; 112 } 113 write("views/build_git.json", JSONValue(git).toString()); 114 } 115 116 foreach(arg ; args) { 117 switch(arg.toLower()) { 118 case "--no-plugins": 119 plugins = false; 120 break; 121 case "--portable": 122 portable = true; 123 break; 124 case "default": 125 case "classic": 126 case "allinone": 127 case "all-in-one": 128 type = Type.default_; 129 break; 130 case "hub": 131 type = Type.hub; 132 break; 133 case "node": 134 type = Type.node; 135 break; 136 default: 137 break; 138 } 139 } 140 141 bool[string] active_plugins; 142 143 if(exists("../build-plugins.toml")) { 144 try { 145 foreach(key, value; parseTOML(cast(string)read("../build-plugins.toml"))) { 146 active_plugins[key] = value.type == TOML_TYPE.TRUE; 147 } 148 } catch(TOMLException) {} 149 } 150 151 TOMLDocument[string] plugs; // plugs[location] = settingsfile 152 153 if(plugins) { 154 155 bool loadPlugin(string path) { 156 if(!path.endsWith(dirSeparator)) path ~= dirSeparator; 157 foreach(pack ; ["plugin.toml", "plugin.json"]) { 158 if(exists(path ~ pack)) { 159 if(pack.endsWith(".toml")) { 160 auto toml = parseTOML(cast(string)read(path ~ pack)); 161 toml["single"] = false; 162 plugs[path] = toml; 163 return true; 164 } else { 165 auto json = parseJSON(cast(string)read(path ~ pack)); 166 if(json.type == JSON_TYPE.OBJECT) { 167 json["single"] = false; 168 plugs[path] = TOMLDocument(toTOML(json).table); 169 return true; 170 } 171 } 172 } 173 } 174 return false; 175 } 176 177 void loadZippedPlugin(string path) { 178 // unzip and load as normal plugin 179 auto data = read(path); 180 auto zip = new ZipArchive(data); 181 immutable name = path[path.lastIndexOf("/")+1..$-4]; 182 immutable dest = ".selery/plugins/" ~ name ~ "/"; 183 bool update = true; 184 if(exists(dest)) { 185 if(exists(dest ~ "crc32.json")) { 186 update = false; 187 auto json = parseJSON(cast(string)read(dest ~ "crc32.json")).object; 188 // compare file names 189 if(sort(json.keys).release() != sort(zip.directory.keys).release()) update = true; 190 else { 191 // compare file's crc32 192 foreach(name, member; zip.directory) { 193 if(member.crc32 != json[name].integer) { 194 update = true; 195 break; 196 } 197 } 198 } 199 } 200 if(update) { 201 foreach(string file ; dirEntries(dest, SpanMode.breadth)) { 202 if(file.isFile) remove(file); 203 } 204 } 205 } else { 206 mkdirRecurse(dest); 207 } 208 if(update) { 209 JSONValue[string] files; 210 foreach(name, member; zip.directory) { 211 files[name] = member.crc32; 212 if(!name.endsWith("/")) { 213 zip.expand(member); 214 if(name.indexOf("/") != -1) mkdirRecurse(dest ~ name[0..name.lastIndexOf("/")]); 215 write(dest ~ name, member.expandedData); 216 } 217 } 218 write(dest ~ "crc32.json", JSONValue(files).toString()); 219 } 220 if(!loadPlugin(dest)) loadPlugin(dest ~ name); 221 } 222 223 void loadSinglePlugin(string location) { 224 immutable name = location[location.lastIndexOf("/")+1..$-2].replace("-", "_"); 225 foreach(line ; split(cast(string)read(location), "\n")) { 226 if(line.strip.startsWith("module") && line[6..$].strip.startsWith(name ~ ";")) { 227 string main = name ~ "."; 228 bool uppercase = true; 229 foreach(c ; name) { 230 if(c == '_') { 231 uppercase = true; 232 } else { 233 if(uppercase) main ~= toUpper("" ~ c); 234 else main ~= c; 235 uppercase = false; 236 } 237 } 238 plugs[location] = TOMLDocument(["name": TOMLValue(name.replace("_", "-")), "main": TOMLValue(main)]); 239 break; 240 } 241 } 242 } 243 244 writeln("Generating dub package for ", Software.name, " ", Software.displayVersion, "."); 245 246 // load plugins in plugins folder 247 if(exists("../plugins")) { 248 foreach(string ppath ; dirEntries("../plugins/", SpanMode.shallow)) { 249 if(ppath.isDir) { 250 loadPlugin(ppath); 251 } else if(ppath.isFile && ppath.endsWith(".zip")) { 252 loadZippedPlugin(ppath); 253 } else if(ppath.isFile && ppath.endsWith(".d")) { 254 loadSinglePlugin(ppath); 255 } 256 } 257 } 258 259 } 260 261 Plugin[string] info; 262 263 foreach(path, value; plugs) { 264 Plugin plugin; 265 plugin.name = value["name"].str; 266 checkName(plugin.name); 267 if(path.isFile) { 268 plugin.single = buildNormalizedPath(absolutePath(path)); 269 } 270 plugin.path = buildNormalizedPath(absolutePath(path)); 271 if(plugin.name !in info) { 272 plugin.toml = value; 273 if(!plugin.path.endsWith(dirSeparator)) plugin.path ~= dirSeparator; 274 auto priority = "priority" in value; 275 if(priority) { 276 if(priority.type == TOML_TYPE.STRING) { 277 immutable p = priority.str.toLower; 278 plugin.priority = (p == "high" || p == "🔥") ? 10 : (p == "medium" || p == "normal" ? 5 : 1); 279 } else if(priority.type == TOML_TYPE.INTEGER) { 280 plugin.priority = clamp(priority.integer.to!size_t, 1, 10); 281 } 282 } 283 auto authors = "authors" in value; 284 auto author = "author" in value; 285 if(authors && authors.type == TOML_TYPE.ARRAY) { 286 foreach(a ; authors.array) { 287 if(a.type == TOML_TYPE.STRING) { 288 plugin.authors ~= a.str; 289 } 290 } 291 } else if(author && author.type == TOML_TYPE.STRING) { 292 plugin.authors = [author.str]; 293 } 294 auto target = "target" in value; 295 if(target && target.type == TOML_TYPE.STRING) { 296 switch(target.str.toLower) { 297 case "default": 298 plugin.target = Type.default_; 299 break; 300 case "hub": 301 plugin.target = Type.hub; 302 break; 303 case "node": 304 plugin.target = Type.node; 305 break; 306 default: 307 break; 308 } 309 } 310 foreach(mname, mvalue; (plugin.target == Type.default_ ? ["hub-main": Type.hub, "node-main": Type.node] : ["main": plugin.target])) { 311 auto mptr = mname in value; 312 if(mptr && mptr.type == TOML_TYPE.STRING) { 313 Main main; 314 string[] spl = mptr.str.split("."); 315 if(plugin.single.length) { 316 main.module_ = spl[0]; 317 main.main = mptr.str; 318 } else { 319 immutable m = mptr.str.lastIndexOf("."); 320 if(m != -1) { 321 main.module_ = mptr.str[0..m]; 322 main.main = mptr.str; 323 } 324 } 325 plugin.main[mvalue] = main; 326 } 327 } 328 if(plugin.single.length) { 329 plugin.version_ = "~single"; 330 } else { 331 foreach(string file ; dirEntries(plugin.path ~ "src", SpanMode.breadth)) { 332 if(file.isFile && file.endsWith(dirSeparator ~ "api.d")) { 333 plugin.api = true; 334 break; 335 } 336 } 337 if(exists(plugin.path ~ ".git") && isDir(plugin.path ~ ".git")) { 338 // try to get version using git 339 immutable tag = executeShell("cd " ~ plugin.path ~ " && git describe --tags").output.strip; //TODO do not use && 340 if(tag.startsWith("v")) plugin.version_ = tag; 341 } 342 } 343 info[plugin.name] = plugin; 344 } else { 345 throw new Exception("Plugin '" ~ plugin.name ~ "' at " ~ plugin.path ~ " conflicts with a plugin with the same name at " ~ info[plugin.name].path); 346 } 347 } 348 349 // remove plugins disabled in plugins.toml 350 foreach(plugin, enabled; active_plugins) { 351 if(!enabled) { 352 auto p = plugin in info; 353 if(p) p.enabled = false; 354 } 355 } 356 357 auto ordered = info.values; 358 359 // sort by priority (or alphabetically) 360 sort!"a.priority == b.priority ? a.name < b.name : a.priority > b.priority"(ordered); 361 362 // control api version 363 foreach(ref inf ; ordered) { 364 long[] api; 365 auto ptr = "api" in inf.toml; 366 if(ptr) { 367 if((*ptr).type == TOML_TYPE.INTEGER) { 368 api ~= (*ptr).integer; 369 } else if((*ptr).type == TOML_TYPE.ARRAY) { 370 foreach(v ; (*ptr).array) { 371 if(v.type == TOML_TYPE.INTEGER) api ~= v.integer; 372 } 373 } else if((*ptr).type == TOML_TYPE.TABLE) { 374 auto from = "from" in *ptr; 375 auto to = "to" in *ptr; 376 if(from && (*from).type == TOML_TYPE.INTEGER && to && (*to).type == TOML_TYPE.INTEGER) { 377 foreach(a ; (*from).integer..(*to).integer+1) { 378 api ~= a; 379 } 380 } 381 } 382 } 383 if(api.length == 0 /*|| api.canFind(Software.api)*/) { 384 writeln(inf.name, " ", inf.version_, ": loaded"); 385 } else { 386 writeln(inf.name, " ", inf.version_, ": cannot load due to wrong api ", api); 387 return 1; 388 } 389 } 390 391 JSONValue[string] builder; 392 builder["name"] = "selery-builder"; 393 builder["targetName"] = "selery" ~ (type!=Type.default_ ? "-" ~ type : "") ~ (portable ? "-" ~ Software.displayVersion : ""); 394 builder["targetType"] = "executable"; 395 builder["targetPath"] = ".."; 396 builder["workingDirectory"] = ".."; 397 builder["sourceFiles"] = ["main/" ~ type ~ ".d", ".selery/builder.d"]; 398 builder["dependencies"] = [ 399 "selery": ["path": ".."], 400 "toml": ["version": "~>1.0.0-rc.3"], 401 "toml:json": ["version": "~>1.0.0-rc.3"] 402 ]; 403 builder["configurations"] = [["name": cast(string)type]]; 404 builder["subPackages"] = new JSONValue[0]; 405 406 string loads = ""; 407 408 if(!exists(".selery")) mkdir(".selery"); 409 410 string[] pluginsFile; 411 412 JSONValue[] json; 413 414 foreach(ref value ; ordered) { 415 pluginsFile ~= value.name ~ " = " ~ value.enabled.to!string; 416 if(value.enabled) { 417 if(value.single.length) { 418 builder["sourceFiles"].array ~= JSONValue(relativePath(value.single)); 419 } else { 420 JSONValue[string] sub; 421 sub["name"] = value.name; 422 sub["targetType"] = "library"; 423 sub["targetPath"] = ".." ~ dirSeparator ~ "libs"; 424 sub["configurations"] = [["name": "plugin"]]; 425 sub["dependencies"] = ["selery": ["path": ".."]], 426 sub["sourcePaths"] = [relativePath(value.path ~ "src")]; 427 sub["importPaths"] = [relativePath(value.path ~ "src")]; 428 auto dptr = "dependencies" in value.toml; 429 if(dptr && dptr.type == TOML_TYPE.TABLE) { 430 foreach(name, d; dptr.table) { 431 if(name.startsWith("dub:")) { 432 sub["dependencies"][name[4..$]] = toJSON(d); 433 } else if(name == "dub" && d.type == TOML_TYPE.TABLE) { 434 foreach(dname, dd; d.table) { 435 sub["dependencies"][dname] = toJSON(dd); 436 } 437 } else { 438 //TODO depends on another plugin 439 sub["dependencies"][":" ~ name] = "*"; 440 } 441 } 442 } 443 auto subConfigurations = "subConfigurations" in value.toml; 444 if(subConfigurations && subConfigurations.type == TOML_TYPE.TABLE) { 445 sub["subConfigurations"] = toJSON(*subConfigurations); 446 } 447 builder["subPackages"].array ~= JSONValue(sub); 448 builder["dependencies"][":" ~ value.name] = "*"; 449 } 450 foreach(string mname; value.target==Type.default_ ? [Type.hub, Type.node] : [value.target]) { 451 auto main = mname in value.main; 452 string load = "ret ~= new PluginOf!(" ~ (main ? main.main : "Object") ~ ")(`" ~ value.name ~ "`, `" ~ value.path ~ "`, " ~ value.authors.to!string ~ ", `" ~ value.version_ ~ "`);"; 453 auto when = "when" in value.toml; 454 if(when && when.type == TOML_TYPE.STRING) { 455 load = "if(" ~ when.str ~ "){ " ~ load ~ " }"; 456 } 457 if(value.single.length) load = "static if(is(" ~ value.main[Type.default_].main ~ " == class)){ " ~ load ~ " }"; 458 if(main) load = "static import " ~ main.module_ ~ "; " ~ load; 459 auto staticWhen = "static-when" in value.toml; 460 if(staticWhen && staticWhen.type == TOML_TYPE.STRING) { 461 load = "static if(" ~ staticWhen.str ~ "){ " ~ load ~ " }"; 462 } 463 load = "static if(target == `" ~ mname ~ "`){ " ~ load ~ " }"; 464 loads ~= "\t" ~ load ~ "\n"; 465 } 466 json ~= value.toJSON(); 467 if(portable) { 468 // copy plugins/$plugin/assets into assets/plugins/$plugin 469 immutable assets = value.path ~ "assets" ~ dirSeparator; 470 if(exists(assets) && assets.isDir) { 471 foreach(file ; dirEntries(assets, SpanMode.breadth)) { 472 immutable dest = "../assets/plugins" ~ dirSeparator ~ value.name ~ dirSeparator ~ file[assets.length..$]; 473 if(file.isFile) { 474 mkdirRecurse(dest[0..dest.lastIndexOf(dirSeparator)]); 475 write(dest, read(file)); 476 } 477 } 478 } 479 } 480 } 481 } 482 483 writeDiff(".selery/builder.d", "module pluginloader;\n\nimport selery.about : Software;\nimport selery.config : Config;\nimport selery.plugin : Plugin;\n\nPlugin[] loadPlugins(alias PluginOf, string target)(inout Config config){\n\tPlugin[] ret;\n" ~ loads ~ "\treturn ret;\n}\n\nenum info = `" ~ JSONValue(json).toString() ~ "`;\n"); 484 485 writeDiff("dub.json", JSONValue(builder).toString()); 486 487 write("../build-plugins.toml", pluginsFile.join(newline) ~ newline); 488 489 if(portable) { 490 491 auto zip = new ZipArchive(); 492 493 // get all files in assets 494 foreach(string file ; dirEntries("../assets/", SpanMode.breadth)) { 495 immutable name = file[10..$].replace("\\", "/"); 496 if(file.isFile && !name.startsWith(".") && !name.endsWith(".ico") && (!name.startsWith("web/") || name.endsWith("/main.css") || name.indexOf("/res/") != -1)) { 497 //TODO optimise .lang files by removing empty lines, windows' line endings and comments 498 auto data = read(file); 499 auto member = new ArchiveMember(); 500 member.name = name; 501 member.expandedData(cast(ubyte[])(file.endsWith(".json") ? parseJSON(cast(string)data).toString() : data)); 502 member.compressionMethod = CompressionMethod.deflate; 503 zip.addMember(member); 504 } 505 } 506 mkdirRecurse("views"); 507 write("views/portable.zip", zip.build()); 508 509 } else if(exists("views/portable.zip")) { 510 511 remove("views/portable.zip"); 512 513 } 514 515 return 0; 516 517 } 518 519 enum invalid = ["selery", "sel", "toml", "default", "lite", "hub", "node", "builder", "condition", "config", "starter", "pluginloader"]; 520 521 void checkName(string name) { 522 void error(string message) { 523 throw new Exception("Cannot load plugin '" ~ name ~ "': " ~ message); 524 } 525 if(name.matchFirst(ctRegex!`[^a-z0-9\-]`)) error("Name contains characters outside the range a-z0-9-"); 526 if(name.length == 0 || name.length > 64) error("Invalid name length: " ~ name.length.to!string ~ " is not between 1 and 64"); 527 if(invalid.canFind(name)) error("Name is reserved"); 528 } 529 530 void writeDiff(string location, const void[] data) { 531 if(!exists(location) || read(location) != data) write(location, data); 532 } 533 534 struct Plugin { 535 536 bool enabled = true; 537 bool api = false; 538 539 TOMLDocument toml; 540 541 string single; 542 543 size_t priority = 5; 544 string path; 545 546 string name; 547 string[] authors; 548 string version_ = "~local"; 549 550 Type target = Type.default_; 551 Main[Type] main; 552 553 JSONValue toJSON() { 554 JSONValue[string] ret; 555 ret["path"] = path; 556 ret["name"] = name; 557 ret["authors"] = authors; 558 ret["version"] = version_; 559 ret["target"] = target; 560 if(main.length) { 561 JSONValue[string] mret; 562 foreach(key, value; main) { 563 mret[key] = value.toJSON(); 564 } 565 ret["main"] = mret; 566 } 567 return JSONValue(ret); 568 } 569 570 } 571 572 struct Main { 573 574 string module_; 575 string main; 576 577 JSONValue toJSON() { 578 return JSONValue([ 579 "module": module_, 580 "main": main 581 ]); 582 } 583 584 }