optimize git clone by reusing existing copies of the same repo

Author: medvednikov

github.v

@@ -80,23 +80,56 @@ fn parse_github_timestamp(s string) int {
 
 fn parse_github_owner_repo(clone_url string) ?(string, string) {
 	mut s := clone_url.trim_space()
+	mut lower := s.to_lower()
+	if lower.starts_with('ssh://') {
+		s = s['ssh://'.len..]
+		lower = s.to_lower()
+	}
 	for prefix in ['https://', 'http://', 'git@'] {
-		if s.starts_with(prefix) {
+		if lower.starts_with(prefix) {
 			s = s[prefix.len..]
+			lower = s.to_lower()
 			break
 		}
 	}
-	s = s.trim_string_left('github.com')
+	if lower.starts_with('git@') {
+		s = s['git@'.len..]
+		lower = s.to_lower()
+	}
+	if lower.starts_with('www.') {
+		s = s['www.'.len..]
+		lower = s.to_lower()
+	}
+	if !lower.starts_with('github.com') {
+		return none
+	}
+	s = s['github.com'.len..]
+	if s == '' || !(s[0] == `/` || s[0] == `:`) {
+		return none
+	}
 	s = s.trim_left(':/')
-	s = s.trim_string_right('.git')
+	if idx := s.index('?') {
+		s = s[..idx]
+	}
+	if idx := s.index('#') {
+		s = s[..idx]
+	}
 	s = s.trim('/')
+	if s.ends_with('.git') {
+		s = s[..s.len - '.git'.len]
+	}
 	parts := s.split('/')
 	if parts.len < 2 || parts[0] == '' || parts[1] == '' {
 		return none
 	}
 	return parts[0], parts[1]
 }
 
+fn is_github_clone_url(clone_url string) bool {
+	parse_github_owner_repo(clone_url) or { return false }
+	return true
+}
+
 // Returns the local user id for a GitHub login, creating an unregistered
 // "shadow" user (no password, no email, just the username and GitHub avatar)
 // when one does not yet exist.
@@ -303,6 +336,11 @@ fn (mut app App) find_or_create_github_shadow_contributor(github_login string, a
 
 fn (mut app App) import_github_issues(repo_id int, clone_url string, owner_user_id int) ! {
 	eprintln('[github-import] starting for repo_id=${repo_id} clone_url=${clone_url} owner_user_id=${owner_user_id}')
+	defer {
+		app.sync_repo_open_issue_count(repo_id) or {
+			eprintln('[github-import] cannot sync issue count: ${err}')
+		}
+	}
 	owner, name := parse_github_owner_repo(clone_url) or {
 		eprintln('[github-import] ERROR: cannot parse github url: ${clone_url}')
 		return error('cannot parse github url: ${clone_url}')

gitly.v

@@ -437,10 +437,17 @@ fn (mut app App) migrate_tables() ! {
 	app.add_missing_column('Repo', 'disable_milestones', db_bool_column_type())!
 	app.add_missing_column('Repo', 'disable_wiki', db_bool_column_type())!
 	app.add_missing_column('Repo', 'is_pinned', db_bool_column_type())!
+	app.add_missing_column('Repo', 'created_at', 'INTEGER NOT NULL DEFAULT 0')!
+	app.backfill_repo_created_at()!
 
 	app.db.exec('create index if not exists idx_commit_repo_created on ${sql_table('Commit')} (repo_id, created_at desc)')!
 }
 
+fn (mut app App) backfill_repo_created_at() ! {
+	created_at := int(time.now().unix())
+	app.db.exec('update ${sql_table('Repo')} set ${sql_table('created_at')} = ${created_at} where ${sql_table('created_at')} is null or ${sql_table('created_at')} <= 0')!
+}
+
 fn (mut app App) add_missing_column(table_name string, column_name string, column_type string) ! {
 	if db_column_exists(mut app.db, table_name, column_name)! {
 		return

issue.v

@@ -131,16 +131,32 @@ fn (mut app App) find_issue_by_id(issue_id int) ?Issue {
 fn (mut app App) find_repo_issues_as_page(repo_id int, page int) []Issue {
 	off := page * commits_per_page
 	return sql app.db {
-		select from Issue where repo_id == repo_id && is_pr == false limit 35 offset off
+		select from Issue where repo_id == repo_id && is_pr == false order by created_at desc limit commits_per_page offset off
 	} or { []Issue{} }
 }
 
 fn (mut app App) get_repo_issue_count(repo_id int) int {
 	return sql app.db {
-		select count from Issue where repo_id == repo_id
+		select count from Issue where repo_id == repo_id && is_pr == false
 	} or { 0 }
 }
 
+fn (mut app App) sync_repo_open_issue_count(repo_id int) ! {
+	open_issues_count := app.get_repo_issue_count(repo_id)
+	sql app.db {
+		update Repo set nr_open_issues = open_issues_count where id == repo_id
+	}!
+}
+
+fn placeholder_user(user_id int) User {
+	username := if user_id > 0 { 'user-${user_id}' } else { 'unknown-user' }
+	return User{
+		id:       user_id
+		username: username
+		avatar:   default_avatar_name
+	}
+}
+
 fn (mut app App) find_user_issues(user_id int) []Issue {
 	return sql app.db {
 		select from Issue where author_id == user_id && is_pr == false order by created_at desc

issue_routes.v

@@ -58,8 +58,11 @@ pub fn (mut app App) handle_add_repo_issue(mut ctx Context, username string, rep
 		return ctx.redirect('/${username}/${repo_name}/issues/new')
 	}
 	app.increment_user_post(mut ctx.user) or { app.info(err.str()) }
-	app.add_issue(repo.id, ctx.user.id, title, text) or { app.info(err.str()) }
-	app.increment_repo_issues(repo.id) or { app.info(err.str()) }
+	app.add_issue(repo.id, ctx.user.id, title, text) or {
+		app.info(err.str())
+		return ctx.redirect('/${username}/${repo_name}/issues/new')
+	}
+	app.sync_repo_open_issue_count(repo.id) or { app.info(err.str()) }
 	app.dispatch_webhook(repo.id, 'issue', WebhookIssuePayload{
 		action: 'opened'
 		repo:   '${username}/${repo_name}'
@@ -80,16 +83,33 @@ pub fn (mut app App) handle_get_repo_issues(mut ctx Context, username string, re
 
 @['/:username/:repo_name/issues/:page']
 pub fn (mut app App) issues(mut ctx Context, username string, repo_name string, page string) veb.Result {
-	repo := app.find_repo_by_name_and_username(repo_name, username) or { return ctx.not_found() }
-	page_i := page.int()
+	mut repo := app.find_repo_by_name_and_username(repo_name, username) or {
+		return ctx.not_found()
+	}
+	mut page_i := page.int()
+	if page_i < 0 {
+		page_i = 0
+	}
+	issue_count := app.get_repo_issue_count(repo.id)
+	if repo.nr_open_issues != issue_count {
+		app.sync_repo_open_issue_count(repo.id) or { app.info(err.str()) }
+		repo.nr_open_issues = issue_count
+	}
+	page_count := calculate_pages(issue_count, commits_per_page)
+	if page_i > page_count {
+		if page_count == 0 {
+			return ctx.redirect('/${repo.user_name}/${repo.name}/issues')
+		}
+		return ctx.redirect('/${repo.user_name}/${repo.name}/issues/${page_count}')
+	}
 	mut issues_with_users := []IssueWithUser{}
 	mut issue := Issue{}
 	mut user := User{}
 	repo_issues := app.find_repo_issues_as_page(repo.id, page_i)
 	mut i := 0
 	for i = 0; i < repo_issues.len; i++ {
 		issue = repo_issues[i]
-		user = app.get_user_by_id(issue.author_id) or { continue }
+		user = app.get_user_by_id(issue.author_id) or { placeholder_user(issue.author_id) }
 		issue.labels = app.get_issue_labels(issue.id)
 		issue.repo_author = repo.user_name
 		issue.repo_name = repo.name
@@ -99,23 +119,8 @@ pub fn (mut app App) issues(mut ctx Context, username string, repo_name string,
 		}
 	}
 	show_repo_link := false
-	mut first := false
-	mut last := false
-	if repo.nr_open_issues > commits_per_page {
-		offset := page_i * commits_per_page
-		delta := repo.nr_open_issues - offset
-		if delta > 0 {
-			if delta == repo.nr_open_issues && page_i == 0 {
-				first = true
-			} else {
-				last = true
-			}
-		}
-	} else {
-		last = true
-		first = true
-	}
-	page_count := calculate_pages(repo.nr_open_issues, commits_per_page)
+	first := page_i == 0
+	last := page_i >= page_count
 	prev_page, next_page := generate_prev_next_pages(page_i)
 	ctx.set_page_title(['Issues', '${repo.user_name}/${repo.name}'])
 	return $veb.html()
@@ -125,7 +130,10 @@ pub fn (mut app App) issues(mut ctx Context, username string, repo_name string,
 pub fn (mut app App) issue(mut ctx Context, username string, repo_name string, id string) veb.Result {
 	repo := app.find_repo_by_name_and_username(repo_name, username) or { return ctx.not_found() }
 	issue := app.find_issue_by_id(id.int()) or { return ctx.not_found() }
-	issue_author := app.get_user_by_id(issue.author_id) or { return ctx.not_found() }
+	if issue.repo_id != repo.id || issue.is_pr {
+		return ctx.not_found()
+	}
+	issue_author := app.get_user_by_id(issue.author_id) or { placeholder_user(issue.author_id) }
 	ctx.set_page_title(['${issue.title} #${issue.id}', '${repo.user_name}/${repo.name}'])
 	mut comments_with_users := []CommentWithUser{}
 	mut comment := Comment{}
@@ -134,7 +142,9 @@ pub fn (mut app App) issue(mut ctx Context, username string, repo_name string, i
 	mut i := 0
 	for i = 0; i < issue_comments.len; i++ {
 		comment = issue_comments[i]
-		comment_author = app.get_user_by_id(comment.author_id) or { continue }
+		comment_author = app.get_user_by_id(comment.author_id) or {
+			placeholder_user(comment.author_id)
+		}
 		comments_with_users << CommentWithUser{
 			item: comment
 			user: comment_author
@@ -176,7 +186,7 @@ pub fn (mut app App) user_issues(mut ctx Context, username string, tab string) v
 	}
 	mut issues_with_users := []IssueWithUser{}
 	for issue in issues {
-		issue_author := app.get_user_by_id(issue.author_id) or { continue }
+		issue_author := app.get_user_by_id(issue.author_id) or { placeholder_user(issue.author_id) }
 		issues_with_users << IssueWithUser{
 			item: issue
 			user: issue_author