Ok here's a small sample that I'm perplexed by:

(ns my-app.my-ns
  (:require [malli.core :as m]
            [malli.dot :as dot]))

(def comment-schema
  {::id         :uuid
   ::content    :string
   ::replies    [:vector [:ref ::comment]]
   ::updated-at inst?
   ::created-at inst?
   ::comment    [:map
                  ::id
                  ::content
                  [::replies {:optional true}]
                  ::updated-at
                  ::created-at]})

(def reg {:registry (merge (m/default-schemas) comment-schema)})

(def c
  {::id         #uuid"ff43dad0-8007-47d7-845e-0b20a021bedb"
   ::content    "hello"
   ::created-at (java.time.Instant/now)
   ::updated-at (java.time.Instant/now)})

(m/validate (m/schema ::comment reg) c)
; => true

(m/schema ::comment reg)
; => ::comment

(m/schema [:schema reg ::comment] reg)
; =>
; Execution error (ExceptionInfo) at malli.impl.util/-fail! (util.cljc:17).
; :malli.core/child-error {:type :enum, :properties nil, :children nil, :min 1}

(dot/transform (m/schema ::comment reg))
; =>
; Execution error (ExceptionInfo) at malli.impl.util/-fail! (util.cljc:17).
; :malli.core/invalid-schema {:schema ::comment}

;; trace:
                             malli.dot/transform                dot.cljc:   58
                             malli.dot/transform                dot.cljc:   60
                              malli.dot/-collect                dot.cljc:   17
                                 malli.core/walk               core.cljc: 1655
                                 malli.core/walk               core.cljc: 1657
     malli.core/-schema-schema/reify/reify/-walk               core.cljc: 1276
                    malli.core/walk/reify/-outer               core.cljc: 1662
                           malli.dot/-collect/fn                dot.cljc:   21
              malli.core/-properties-and-options               core.cljc:  270
                   malli.core/-property-registry               core.cljc:  265
                          clojure.core/reduce-kv                core.clj: 6856
                     clojure.core.protocols/fn/G           protocols.clj:  175
                                 clojure.core/fn                core.clj: 6845
                                             ...                              
                malli.core/-property-registry/fn               core.cljc:  265
                               malli.core/schema               core.cljc: 1610
                              malli.core/-schema               core.cljc:  251
                          malli.impl.util/-fail!               util.cljc:   16

(dot/transform (m/schema ::comment reg) reg)
; =>
; Execution error (ExceptionInfo) at malli.impl.util/-fail! (util.cljc:17).
; :malli.core/invalid-schema {:schema ::comment}
;; same trace as above

This one also fails..

(def cons-schema {::cons [:maybe [:tuple pos-int? [:ref ::cons]]]})
(def cons-list [1 [2 [3 [4 [5 nil]]]]])
(def reg {:registry (merge (m/default-schemas) cons-schema)})
(dot/transform (m/schema ::cons reg) reg)
; => 
; :malli.core/invalid-schema {:schema ::cons}

but this works: (m/validate (m/schema ::cons reg) cons-list) => true

Following up on this, made some further investigations. this works:

(dot/transform [:schema {:registry cons-schema} ::cons])

(dot/transform [:schema {:registry (merge (m/default-schemas) cons-schema)} ::cons])
; =>
:malli.core/child-error {:type :enum, :properties nil, :children nil, :min 1}

(m/schema [:schema {:registry cons-schema} ::cons]) ;; works
(m/schema ::cons {:registry cons-schema}) ;; fails -> :malli.core/invalid-schema {:schema :maybe}