rm temp

crawler/gitlab.sh

@@ -32,7 +32,6 @@ gitlab() (
       blob="$(urlencode "$blob")"
 
       local path="api/v4/projects/$project/repository/files/$blob/raw"
-		log "url: https://gitlab-app.eng.qops.net/$path (blob=$blob, project=$project)"
       echo "https://gitlab-app.eng.qops.net/$path"
 	}
 
@@ -73,7 +72,6 @@ gitlab() (
       root="${root#tree/}"
 		root="$(echo "$root" | sed 's/^[^\/]*//')"
 		root="${root#/}"
-		log project=$project, root=$root, url=$url
       echo "$project"
       echo "$root"
    }

crawler/gitlab_wiki.sh

@@ -6,8 +6,9 @@ gitlab_wiki() (
    }
 
 	human_url() {
-		log "not impl: human url: $@"
-		exit 1
+		local url="${1%/}"
+		url="${url%%#*}"
+		echo "$url/$(echo "$2" | base64 --decode)"
 	}
 
 	_host() {
@@ -25,7 +26,7 @@ gitlab_wiki() (
 		project="${project%/-/}"
 		project="${project#/}"
 		project="${project%/}"
-		echo "$project"
+		echo "${project%%#*}"
 	}
 
 	_blob() {
@@ -36,7 +37,7 @@ gitlab_wiki() (
 		local blob="${path#*/wikis}"
 		blob="${blob#/}"
 		blob="${blob%/}"
-		echo "$blob"
+		echo "${blob%%#*}"
 	}
 
    get() {

crawler/main.sh

@@ -95,6 +95,8 @@ crawl_with() {
 	local content="$(echo "$json" | jq -r .content)"
 	local crawlable_source="$(extract_crawlable_source "$content")"
 
+	notes put "$pid" "$(notes meta "$pid" | jq -r .Meta.Title)" "$crawlable_source"
+
 	local expanded=($($backend expand "$crawlable_source"))
 
 	log purge $crawlable_source:
@@ -102,7 +104,7 @@ crawl_with() {
 		notes del "$subid"
 	done
 
-	log expand $crawlable_source:"$expanded"
+	log expand $crawlable_source:"${#expanded[@]}: ${expanded[@]}"
 	notes_mkdir_p() {
 		local id="$1"
 		local subtitle="${2%/}"
@@ -148,8 +150,9 @@ crawl_with() {
 			echo "$sum"
 		)"
 		ID="${ID%/}"
-		if [ "${#expanded[@]}" == 1 ]; then
+		if [ "${#expanded[@]}" -lt 2 ]; then
 			ID="$pid"
+			TITLE="$(notes meta "$ID" | jq -r .Meta.Title)"
 			CONTENT="$(printf "%s\n\n%s", "$crawlable_source" "$CONTENT")"
 		fi
 		log "	$ID ($TITLE): ${#CONTENT}"

server/go.mod

@@ -3,8 +3,8 @@ module ezmded
 go 1.17
 
 require (
+	github.com/gomarkdown/markdown v0.0.0-20220114203417-14399d5448c4
 	github.com/google/uuid v1.3.0
-	go.mongodb.org/mongo-driver v1.7.2
 	gopkg.in/yaml.v2 v2.4.0
 	local/args v0.0.0-00010101000000-000000000000
 	local/gziphttp v0.0.0-00010101000000-000000000000
@@ -12,11 +12,6 @@ require (
 	local/simpleserve v0.0.0-00010101000000-000000000000
 )
 
-require (
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/gomarkdown/markdown v0.0.0-20220114203417-14399d5448c4 // indirect
-)
-
 replace local/args => ../../../../args
 
 replace local/logb => ../../../../logb

server/go.sum

@@ -18,14 +18,12 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/coreos/bbolt v0.0.0-20180318001526-af9db2027c98/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/djherbis/times v1.1.0/go.mod h1:CGMZlo255K5r4Yw0b9RRfFQpM2y7uOmxg4jm9HsaVf8=
 github.com/dropbox/dropbox-sdk-go-unofficial v5.4.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fairlyblank/md2min v0.0.0-20171213131418-39cd6e9904ac/go.mod h1:QAobgT+CwT/SRphqV6Jrz5jt3wkW9Q72QNquEvh6dLk=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
@@ -60,7 +58,6 @@ github.com/gomarkdown/markdown v0.0.0-20210208175418-bda154fe17d8/go.mod h1:aii0
 github.com/gomarkdown/markdown v0.0.0-20220114203417-14399d5448c4 h1:6GlsnS3GQYfrJZTJEUsheoyLE6kLXQJDvQKIKxgL/9Q=
 github.com/gomarkdown/markdown v0.0.0-20220114203417-14399d5448c4/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -115,7 +112,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.8.3/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rfjakob/eme v0.0.0-20171028163933-2222dbd4ba46/go.mod h1:U2bmx0hDj8EyDdcxmD5t3XHDnBFnyNNc22n1R4008eM=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -143,11 +139,9 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/t3rm1n4l/go-mega v0.0.0-20190205172012-55a226cf41da/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
@@ -156,7 +150,6 @@ github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.4-0.20210326114109-75d8cce5b78c/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yunify/qingstor-sdk-go v2.2.15+incompatible/go.mod h1:w6wqLDQ5bBTzxGJ55581UrSwLrsTAsdo9N6yX/8d9RY=
-go.mongodb.org/mongo-driver v1.7.2 h1:pFttQyIiJUHEn50YfZgC9ECjITMT44oiN36uArf/OFg=
 go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
 golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -213,5 +206,4 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

server/public/ui/files.ctmpl

@@ -2,7 +2,7 @@
 <!DOCTYPE html>
 <html>
 	<header>
-		<title>id={{ .This.ID }}, {{ .This.Title }}</title>
+		<title>{{ .This.Title }}</title>
 		{{ template "_import" }}
 	</header>
 	<body class="fullscreen tb_fullscreen lr_fullscreen" style="position: absolute">

server/public/ui/templates/_about.ctmpl

@@ -32,6 +32,16 @@
 			<li>Create a file that just contains "https://gitlab.com/my/repo/-/tree/master/README.md" or "https://docs.google.com/docs/my-doc/edit", wait some time, and now it's an updating version of that doc</li>
 			<li>Create a file that just contains "https://gitlab.com/my/repo/-/tree/master/runbooks", wait some time, and now it's an updating version of all those docs</li>
 		</ul>
+
+		<h3>Butt how do I use it?</h3>
+		<div>
+			<ol>
+				<li>Make or edit a file</li>
+				<li>The first line is a link to Gitlab or Google</li>
+				<li>Save</li>
+				<li>Wait</li>
+			</ol>
+		</div>
 	</div>
 
 	<h2>I got a bone to pick with you!!! Who are you exactly?</h2>

server/public/ui/templates/_editor.ctmpl

@@ -5,10 +5,15 @@
 	#easyMDEwrap {
 		flex-grow: 1;
 	}
+	.CodeMirror {
+		min-height: 7em;
+	}
 	.CodeMirror-scroll, .CodeMirror-sizer {
-		min-height: 10px !important;
 		height: auto !important;
 	}
+	.CodeMirror-sizer {
+		min-height: 10rem !important;
+	}
 	#article {
 		display: flex;
 		flex-direction: column;

todo.yaml

@@ -1,20 +1,22 @@
 todo:
-- /ui/files is an about page over a redir
-- gitlab wiki original links are empty
-- /ui/files does not redir in b1
 - mark generated via meta so other files in the dir can be created, deleted, replaced safely
 - links like `/Smoktests` in user-files home wiki don't rewrite
 - map fullURLScraped->internalURL for relative links sometimes
-- anchors on gitlab wikis at least are bad
-- min-height for easymde
-- use `meta` so no need for extra level for explicit single files
 - scrape odo
 - rewrite links if available to local
-- table of contents
 - anchor per line
 - anchor links work
 - ui; last updated; 2022.02.01T12:34:56
 done:
+- gitlab/-/blob/about.md does NOT map to exactly 1 file
+- crawler does NOT modify title cause readme.md everywhere
+- use `meta` so no need for extra level for explicit single files
+- table of contents
+- min-height for easymde
+- /ui/files does not redir in b1
+- anchors on gitlab wikis at least are bad
+- gitlab wiki original links are empty
+- /ui/files is an about page over a redir
 - use `read-only` for autogenerated things;; could skip easymde and make google docs much faster
 - new line after original link
 - scrape gslide