+ # description.txt
+ if not "description" in result:
+ try:
+ with open(self.baseDir + "/description.txt", "r") as myfile:
+ result["description"] = myfile.read()
+ except IOError:
+ print("description.txt does not exist!")
+
+ # depends.txt
+ import re
+ pattern = re.compile("^([a-z0-9_]+)\??$")
+ if not "depends" in result and not "optional_depends" in result:
+ try:
+ with open(self.baseDir + "/depends.txt", "r") as myfile:
+ contents = myfile.read()
+ soft = []
+ hard = []
+ for line in contents.split("\n"):
+ line = line.strip()
+ if pattern.match(line):
+ if line[len(line) - 1] == "?":
+ soft.append( line[:-1])
+ else:
+ hard.append(line)
+
+ result["depends"] = hard
+ result["optional_depends"] = soft
+
+ except IOError:
+ print("depends.txt does not exist!")
+
+ else:
+ if "depends" in result:
+ result["depends"] = [x.strip() for x in result["depends"].split(",")]
+ if "optional_depends" in result:
+ result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
+
+
+ # Calculate Title
+ if "name" in result and not "title" in result:
+ result["title"] = result["name"].replace("_", " ").title()
+
+ # Calculate short description
+ if "description" in result:
+ desc = result["description"]
+ idx = desc.find(".") + 1
+ cutIdx = min(len(desc), 200 if idx < 5 else idx)
+ result["short_description"] = desc[:cutIdx]
+
+ # Get forum ID
+ info = findModInfo(self.author, result.get("name"), self.repo)
+ if info is not None:
+ result["forumId"] = info.get("topicId")
+
+ if "name" in result:
+ self.name = result["name"]
+ del result["name"]
+
+ self.meta = result
+
+ def addChildrenFromModDir(self, dir):
+ for entry in next(os.walk(dir))[1]:
+ path = dir + "/" + entry
+ if not entry.startswith('.') and os.path.isdir(path):
+ self.children.append(PackageTreeNode(path, name=entry))
+
+
+ def fold(self, attr, key=None, acc=None):
+ if acc is None:
+ acc = set()
+
+ if self.meta is None:
+ return acc
+
+ at = getattr(self, attr)
+ value = at if key is None else at.get(key)
+
+ if isinstance(value, list):
+ acc |= set(value)
+ elif value is not None:
+ acc.add(value)
+
+ for child in self.children:
+ child.fold(attr, key, acc)
+
+ return acc
+
+ def get(self, key):
+ return self.meta.get(key)
+
+def generateGitURL(urlstr):
+ scheme, netloc, path, query, frag = urlsplit(urlstr)
+
+ return "http://:@" + netloc + path + query
+
+# Clones a repo from an unvalidated URL.
+# Returns a tuple of path and repo on sucess.
+# Throws `TaskError` on failure.
+# Caller is responsible for deleting returned directory.
+def cloneRepo(urlstr, ref=None, recursive=False):
+ gitDir = tempfile.gettempdir() + "/" + randomString(10)
+
+ err = None
+ try:
+ gitUrl = generateGitURL(urlstr)
+ print("Cloning from " + gitUrl)
+ repo = git.Repo.clone_from(gitUrl, gitDir, \
+ progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
+
+ if ref is not None:
+ repo.create_head("myhead", ref).checkout()
+ return gitDir, repo
+ except GitCommandError as e:
+ # This is needed to stop the backtrace being weird
+ err = e.stderr
+
+ raise TaskError(err.replace("stderr: ", "") \
+ .replace("Cloning into '" + gitDir + "'...", "") \
+ .strip())
+
+@celery.task()
+def getMeta(urlstr, author):
+ gitDir, _ = cloneRepo(urlstr, recursive=True)
+ tree = PackageTreeNode(gitDir, author=author, repo=urlstr)
+ shutil.rmtree(gitDir)
+
+ result = {}
+ result["name"] = tree.name
+ result["provides"] = tree.fold("name")
+ result["type"] = tree.type.name
+
+ for key in ["depends", "optional_depends"]:
+ result[key] = tree.fold("meta", key)
+
+ for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
+ result[key] = tree.get(key)
+
+ for mod in result["provides"]:
+ result["depends"].discard(mod)
+ result["optional_depends"].discard(mod)
+
+ for key, value in result.items():
+ if isinstance(value, set):
+ result[key] = list(value)