From fd2a82f0f49a043f10ab448ecee1e5acb32c29aa Mon Sep 17 00:00:00 2001 From: cnliao Date: Tue, 9 Oct 2018 10:48:33 +0800 Subject: [PATCH 1/7] fix display(d, m, x) where m expects a json string Fix fredo-dedup/VegaLite.jl#127 After the fix the following works out of the box in new versions of jupyterlab, generating an interactive VegaLite visualization, without additional packages except Ijulia. ``` spec = raw""" { "data": { "values": [ {"a": "A","b": 28}, {"a": "B","b": 55}, {"a": "C","b": 43}, {"a": "D","b": 91}, {"a": "E","b": 81}, {"a": "F","b": 53}, {"a": "G","b": 19}, {"a": "H","b": 87}, {"a": "I","b": 52} ] }, "mark": "point", "encoding": { "x": {"field": "a", "type": "ordinal"}, "y": {"field": "b", "type": "quantitative"} } } """ display("application/vnd.vegalite.v2+json", spec) ``` --- src/inline.jl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/inline.jl b/src/inline.jl index 39e73cf2..e504ed0e 100644 --- a/src/inline.jl +++ b/src/inline.jl @@ -47,6 +47,19 @@ function limitstringmime(mime::MIME, x) return String(take!(buf)) end +const ipy_mime_json = [ + "application/vnd.dataresource+json", + "application/vnd.vegalite.v2+json", + "application/vnd.vega.v3+json", +] +_display_dict(m::MIME, m_str, x) = Dict(m_str=>limitstringmime(m, x)) +# escape JSON string correctly before send_ipython +for mime in ipy_mime_json + @eval begin + _display_dict(m::MIME{Symbol($mime)}, m_str, x) = Dict(m_str=>JSON.JSONText(limitstringmime(m, x))) + end +end + for mime in ipy_mime @eval begin function display(d::InlineDisplay, ::MIME{Symbol($mime)}, x) @@ -54,7 +67,7 @@ for mime in ipy_mime msg_pub(execute_msg, "display_data", Dict( "metadata" => metadata(x), # optional - "data" => Dict($mime => limitstringmime(MIME($mime), x))))) + "data" => _display_dict(MIME($mime), $mime, x)))) end displayable(d::InlineDisplay, ::MIME{Symbol($mime)}) = true end From f9aea3353571544bad9801d413b9abc955dba287 Mon Sep 17 00:00:00 2001 From: cnliao Date: Tue, 9 Oct 2018 11:37:32 +0800 Subject: [PATCH 2/7] add istextmime specialization --- src/inline.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/inline.jl b/src/inline.jl index e504ed0e..850e418c 100644 --- a/src/inline.jl +++ b/src/inline.jl @@ -57,6 +57,7 @@ _display_dict(m::MIME, m_str, x) = Dict(m_str=>limitstringmime(m, x)) for mime in ipy_mime_json @eval begin _display_dict(m::MIME{Symbol($mime)}, m_str, x) = Dict(m_str=>JSON.JSONText(limitstringmime(m, x))) + Base.Multimedia.istextmime(::MIME{Symbol($mime)}) = true end end From 4e3a87e20fa40250ffbe72a48bda21850e3a497d Mon Sep 17 00:00:00 2001 From: cnliao Date: Tue, 9 Oct 2018 13:24:21 +0800 Subject: [PATCH 3/7] jupyterlab 0.34.12 supports vega.v4 rather than v3 Ultimately one should rethink how IJulia should support MIME types. But this fixes things for me now. --- src/inline.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/inline.jl b/src/inline.jl index 850e418c..04f4c08f 100644 --- a/src/inline.jl +++ b/src/inline.jl @@ -8,6 +8,7 @@ const ipy_mime = [ "application/vnd.dataresource+json", "application/vnd.vegalite.v2+json", "application/vnd.vega.v3+json", + "application/vnd.vega.v4+json", "text/html", "text/latex", "image/svg+xml", @@ -51,6 +52,7 @@ const ipy_mime_json = [ "application/vnd.dataresource+json", "application/vnd.vegalite.v2+json", "application/vnd.vega.v3+json", + "application/vnd.vega.v4+json", ] _display_dict(m::MIME, m_str, x) = Dict(m_str=>limitstringmime(m, x)) # escape JSON string correctly before send_ipython From bd4afe7793527d6c61f0d6a69a0fc7b1b4d60771 Mon Sep 17 00:00:00 2001 From: cnliao Date: Wed, 10 Oct 2018 19:42:14 +0800 Subject: [PATCH 4/7] refactor mime support --- src/execute_request.jl | 129 +++++++++++++++++++++++++++++------------ src/inline.jl | 28 ++------- 2 files changed, 99 insertions(+), 58 deletions(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index fe4c3fc3..0f409824 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -5,17 +5,24 @@ import Base.Libc: flush_cstdio import Pkg -const text_plain = MIME("text/plain") -const image_svg = MIME("image/svg+xml") -const image_png = MIME("image/png") -const image_jpeg = MIME("image/jpeg") -const text_markdown = MIME("text/markdown") -const text_html = MIME("text/html") -const text_latex = MIME("text/latex") # Jupyter expects this -const text_latex2 = MIME("application/x-latex") # but this is more standard? -const application_vnd_vega_v3 = MIME("application/vnd.vega.v3+json") -const application_vnd_vegalite_v2 = MIME("application/vnd.vegalite.v2+json") -const application_vnd_dataresource = MIME("application/vnd.dataresource+json") +Base.showable(a::AbstractVector{<:MIME}, x) = any(m -> showable(m, x), a) + +""" +A vector of MIME types (or vectors of MIME types) that IJulia will try to +render. IJulia will try to render every MIME type specified in the first level +of the vector. If a vector of MIME types is specified, IJulia will include only +the first MIME type that is renderable (this allows for the expression of +priority and exclusion of redundant data). + +For example, since "text/plain" is specified as a first-child of the array, +IJulia will always try to include a "text/plain" representation of anything that +is displayed. Since markdown and latex are specified within a sub-vector, IJulia +will always try to render "text/markdown", and will only try to render +"text/latex" if markdown isn't possible. +""" +const ijulia_mime_types = Vector{Union{MIME, AbstractVector{MIME}}}() + +register_mime(x::Union{M, AbstractVector{M}}) where {M <: MIME} = push!(ijulia_mime_types, x) include("magics.jl") @@ -23,34 +30,84 @@ include("magics.jl") # in Jupyter display_data and pyout messages metadata(x) = Dict() -# return a String=>String dictionary of mimetype=>data -# for passing to Jupyter display_data and execute_result messages. -function display_dict(x) - data = Dict{String,Any}("text/plain" => limitstringmime(text_plain, x)) - if showable(application_vnd_vegalite_v2, x) - data[string(application_vnd_vegalite_v2)] = JSON.JSONText(limitstringmime(application_vnd_vegalite_v2, x)) - elseif showable(application_vnd_vega_v3, x) # don't send vega if we have vega-lite - data[string(application_vnd_vega_v3)] = JSON.JSONText(limitstringmime(application_vnd_vega_v3, x)) - end - if showable(application_vnd_dataresource, x) - data[string(application_vnd_dataresource)] = JSON.JSONText(limitstringmime(application_vnd_dataresource, x)) +""" +Generate the preferred MIME representation of x. + +Returns a tuple with the selected MIME type and the representation of the data +using that MIME type. +""" +function display_mimestring(mime_array::AbstractVector{MIME}, x) + for m in mime_array + if showable(mime_array, x) + return display_mimestring(m, x) + end end - if showable(image_svg, x) - data[string(image_svg)] = limitstringmime(image_svg, x) + error("No displayable MIME types in mime array.") +end + +abstract type MIMEStringType end +display_mimestring(m::MIME, x) = display_mimestring(m, mimestringtype(m), x) + +""" + If false then data of the mime type should be sent to ipython in b64-encoded string. + If true then it will be sent as is. + Defaults to the value of istextmime. +""" +_istextmime(m::MIME) = istextmime(m) + +struct RawMIMEString <: MIMEStringType end +for m in [ + MIME("text/plain"), + MIME("image/svg+xml"), + [MIME("image/png"),MIME("image/jpeg")], + [ + MIME("text/markdown"), + MIME("text/html"), + MIME("text/latex"), # Jupyter expects this + MIME("application/x-latex"), # but this is more standard? + ], +] + register_mime(m) + if m isa MIME + @eval mimestringtype(::$m) = RawMIMEString + else + for _m in m + @eval mimestringtype(::$m) = RawMIMEString + end end - if showable(image_png, x) - data[string(image_png)] = limitstringmime(image_png, x) - elseif showable(image_jpeg, x) # don't send jpeg if we have png - data[string(image_jpeg)] = limitstringmime(image_jpeg, x) +end +display_mimestring(m::MIME, ::RawMIMEString, x) = (m, limitstringmime(m, x)) + +struct JSONMIMEString <: MIMEStringType end +for m in [ + [MIME("application/vnd.vegalite.v2+json"), MIME("application/vnd.vega.v3+json")], + MIME("application/vnd.dataresource+json") +] + register_mime(m) + if m <: MIME + @eval mimestringtype(::$m) = JSONMIMEString + @eval _istextmime(::$m) = true + else + for _m in m + @eval mimestringtype(::$m) = JSONMIMEString + @eval _istextmime(::$m) = true + end end - if showable(text_markdown, x) - data[string(text_markdown)] = limitstringmime(text_markdown, x) - elseif showable(text_html, x) - data[string(text_html)] = limitstringmime(text_html, x) - elseif showable(text_latex, x) - data[string(text_latex)] = limitstringmime(text_latex, x) - elseif showable(text_latex2, x) - data[string(text_latex)] = limitstringmime(text_latex2, x) +end +display_mimestring(m::MIME, ::JSONMIMEString, x) = (m, JSON.JSONText(limitstringmime(m, x))) + +""" +Generate a dictionary of `mime_type => data` pairs for all registered MIME +types. This is the format that Jupyter expects in display_data and +execute_result messages. +""" +function display_dict(x) + data = Dict{String, Union{String, JSONText}}() + for m in ijulia_mime_types + if showable(m, x) + mime, mime_repr = display_mimestring(m, x) + data[string(mime)] = mime_repr + end end return data end diff --git a/src/inline.jl b/src/inline.jl index 39e73cf2..920f09f7 100644 --- a/src/inline.jl +++ b/src/inline.jl @@ -2,22 +2,6 @@ import Base: display, redisplay struct InlineDisplay <: AbstractDisplay end -# supported MIME types for inline display in IPython, in descending order -# of preference (descending "richness") -const ipy_mime = [ - "application/vnd.dataresource+json", - "application/vnd.vegalite.v2+json", - "application/vnd.vega.v3+json", - "text/html", - "text/latex", - "image/svg+xml", - "image/png", - "image/jpeg", - "text/plain", - "text/markdown", - "application/javascript" -] - # need special handling for showing a string as a textmime # type, since in that case the string is assumed to be # raw data unless it is text/plain @@ -29,7 +13,7 @@ israwtext(::MIME, x) = false # IOContext that tells the underlying show function to limit output function limitstringmime(mime::MIME, x) buf = IOBuffer() - if istextmime(mime) + if _istextmime(mime) if israwtext(mime, x) return String(x) else @@ -47,14 +31,14 @@ function limitstringmime(mime::MIME, x) return String(take!(buf)) end -for mime in ipy_mime +for mime in ijulia_mime_types @eval begin - function display(d::InlineDisplay, ::MIME{Symbol($mime)}, x) + function display(d::InlineDisplay, m::MIME{Symbol($mime)}, x) send_ipython(publish[], msg_pub(execute_msg, "display_data", Dict( "metadata" => metadata(x), # optional - "data" => Dict($mime => limitstringmime(MIME($mime), x))))) + "data" => Dict($mime=>display_mimestring(m, x)[2])))) end displayable(d::InlineDisplay, ::MIME{Symbol($mime)}) = true end @@ -68,11 +52,11 @@ display(d::InlineDisplay, m::MIME"text/javascript", x) = display(d, MIME("applic # If the user explicitly calls display("foo/bar", x), we send # the display message, also sending text/plain for text data. -displayable(d::InlineDisplay, M::MIME) = istextmime(M) +displayable(d::InlineDisplay, M::MIME) = _istextmime(M) function display(d::InlineDisplay, M::MIME, x) sx = limitstringmime(M, x) d = Dict(string(M) => sx) - if istextmime(M) + if _istextmime(M) d["text/plain"] = sx # directly show text data, e.g. text/csv end send_ipython(publish[], From 1dfd237e9ab1d6a7099b2367c197de1548a3a941 Mon Sep 17 00:00:00 2001 From: cnliao Date: Wed, 10 Oct 2018 22:50:12 +0800 Subject: [PATCH 5/7] improve upon #755 and add new mime for vega4 --- src/execute_request.jl | 77 ++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index 0f409824..408aa38e 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -5,7 +5,7 @@ import Base.Libc: flush_cstdio import Pkg -Base.showable(a::AbstractVector{<:MIME}, x) = any(m -> showable(m, x), a) +Base.showable(a::Vector{<:MIME}, x) = any(m -> showable(m, x), a) """ A vector of MIME types (or vectors of MIME types) that IJulia will try to @@ -20,9 +20,19 @@ is displayed. Since markdown and latex are specified within a sub-vector, IJulia will always try to render "text/markdown", and will only try to render "text/latex" if markdown isn't possible. """ -const ijulia_mime_types = Vector{Union{MIME, AbstractVector{MIME}}}() +const ijulia_mime_types = Vector{Union{MIME, Vector{<:MIME}}}([ + MIME("text/plain"), + MIME("image/svg+xml"), + [MIME("image/png"),MIME("image/jpeg")], + [ + MIME("text/markdown"), + MIME("text/html"), + MIME("text/latex"), # Jupyter expects this + MIME("application/x-latex"), # but this is more standard? + ], +]) -register_mime(x::Union{M, AbstractVector{M}}) where {M <: MIME} = push!(ijulia_mime_types, x) +register_mime(x::Union{M, Vector{M}}) where {M <: MIME} = push!(ijulia_mime_types, x) include("magics.jl") @@ -36,7 +46,15 @@ Generate the preferred MIME representation of x. Returns a tuple with the selected MIME type and the representation of the data using that MIME type. """ -function display_mimestring(mime_array::AbstractVector{MIME}, x) +function display_mimestring end + +abstract type MIMEStringType end +struct RawMIMEString <: MIMEStringType end +mimestringtype(m::MIME) = RawMIMEString +display_mimestring(::Type{T}, m::MIME, x) where{T<:MIMEStringType} = display_mimestring(T(), m, x) +display_mimestring(::RawMIMEString, m::MIME, x) = (m, limitstringmime(m, x)) +display_mimestring(m::MIME, x) = display_mimestring(mimestringtype(m), m, x) +function display_mimestring(mime_array::Vector{<:MIME}, x) for m in mime_array if showable(mime_array, x) return display_mimestring(m, x) @@ -45,56 +63,41 @@ function display_mimestring(mime_array::AbstractVector{MIME}, x) error("No displayable MIME types in mime array.") end -abstract type MIMEStringType end -display_mimestring(m::MIME, x) = display_mimestring(m, mimestringtype(m), x) - """ If false then data of the mime type should be sent to ipython in b64-encoded string. If true then it will be sent as is. Defaults to the value of istextmime. + Use a private name to avoid type piracy. """ _istextmime(m::MIME) = istextmime(m) -struct RawMIMEString <: MIMEStringType end -for m in [ - MIME("text/plain"), - MIME("image/svg+xml"), - [MIME("image/png"),MIME("image/jpeg")], - [ - MIME("text/markdown"), - MIME("text/html"), - MIME("text/latex"), # Jupyter expects this - MIME("application/x-latex"), # but this is more standard? - ], -] - register_mime(m) - if m isa MIME - @eval mimestringtype(::$m) = RawMIMEString - else - for _m in m - @eval mimestringtype(::$m) = RawMIMEString - end - end -end -display_mimestring(m::MIME, ::RawMIMEString, x) = (m, limitstringmime(m, x)) - +""" + To add a new MIME type that require special treatment before sending to ipython, + follow the example of JSONMIMEString to do the following: + 0. define a singleton type inherited from MIMEStringType. This is a trait type. + 1. register_mime + 2. specialize mimestringtype on the new MIME returning the new trait. + 3. (optinal) specialize _istextmime to return true if the new MIME should be send as text. + 4. specialize display_mimestring to implement your special treatment. +""" struct JSONMIMEString <: MIMEStringType end -for m in [ - [MIME("application/vnd.vegalite.v2+json"), MIME("application/vnd.vega.v3+json")], - MIME("application/vnd.dataresource+json") -] - register_mime(m) +for mime in [[MIME("application/vnd.vegalite.v2+json"), MIME("application/vnd.vega.v3+json")], + MIME("application/vnd.vega.v4+json"), + MIME("application/vnd.dataresource+json")] + register_mime(mime) + m = typeof(mime) if m <: MIME @eval mimestringtype(::$m) = JSONMIMEString @eval _istextmime(::$m) = true else - for _m in m + for _m in mime + m = typeof(_m) @eval mimestringtype(::$m) = JSONMIMEString @eval _istextmime(::$m) = true end end end -display_mimestring(m::MIME, ::JSONMIMEString, x) = (m, JSON.JSONText(limitstringmime(m, x))) +display_mimestring(::JSONMIMEString, m::MIME, x) = (m, JSON.JSONText(limitstringmime(m, x))) """ Generate a dictionary of `mime_type => data` pairs for all registered MIME From 92b55f5d2da5c713eed651477d1873764d28a110 Mon Sep 17 00:00:00 2001 From: cnliao Date: Wed, 10 Oct 2018 23:42:13 +0800 Subject: [PATCH 6/7] recklessly nuke workarounds for latex&js in inline.jl --- src/execute_request.jl | 60 +++++++++++++++++++++++++++++------------- src/inline.jl | 22 ---------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index 408aa38e..ec8e1488 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -20,19 +20,43 @@ is displayed. Since markdown and latex are specified within a sub-vector, IJulia will always try to render "text/markdown", and will only try to render "text/latex" if markdown isn't possible. """ -const ijulia_mime_types = Vector{Union{MIME, Vector{<:MIME}}}([ - MIME("text/plain"), - MIME("image/svg+xml"), - [MIME("image/png"),MIME("image/jpeg")], - [ - MIME("text/markdown"), - MIME("text/html"), - MIME("text/latex"), # Jupyter expects this - MIME("application/x-latex"), # but this is more standard? - ], -]) - -register_mime(x::Union{M, Vector{M}}) where {M <: MIME} = push!(ijulia_mime_types, x) +const ijulia_mime_types = Vector{Union{MIME, Vector{<:MIME}}}() + +function register_mime(x::Union{M, Vector{M}}) where {M <: MIME} + push!(ijulia_mime_types, x) + if x isa MIME + register_mime_display(x) + else + register_mime_display.(x) + end +end + +import Base: display, redisplay +struct InlineDisplay <: AbstractDisplay end +function register_mime_display(mime) + mime_type = typeof(mime) + mime_str = string(mime) + @eval begin + function display(d::InlineDisplay, m::$(mime_type), x) + send_ipython(publish[], + msg_pub(execute_msg, "display_data", + Dict( + "metadata" => metadata(x), # optional + "data" => Dict($(mime_str)=>display_mimestring(m, x)[2])))) + end + displayable(d::InlineDisplay, ::MIME{Symbol($mime)}) = true + end +end + +register_mime.([MIME("text/plain"), + MIME("image/svg+xml"), + [MIME("image/png"),MIME("image/jpeg")], + [MIME("text/markdown"), + MIME("text/html"), + MIME("text/latex"), # Jupyter expects this + MIME("application/x-latex")], # but this is more standard? + [MIME("text/javascript"), MIME("application/javascript")], + ]) include("magics.jl") @@ -75,16 +99,15 @@ _istextmime(m::MIME) = istextmime(m) To add a new MIME type that require special treatment before sending to ipython, follow the example of JSONMIMEString to do the following: 0. define a singleton type inherited from MIMEStringType. This is a trait type. - 1. register_mime - 2. specialize mimestringtype on the new MIME returning the new trait. - 3. (optinal) specialize _istextmime to return true if the new MIME should be send as text. - 4. specialize display_mimestring to implement your special treatment. + 1. specialize mimestringtype on the new MIME returning the new trait. + 2. (optinal) specialize _istextmime to return true if the new MIME should be send as text. + 3. specialize display_mimestring to implement your special treatment. + 4. register_mime """ struct JSONMIMEString <: MIMEStringType end for mime in [[MIME("application/vnd.vegalite.v2+json"), MIME("application/vnd.vega.v3+json")], MIME("application/vnd.vega.v4+json"), MIME("application/vnd.dataresource+json")] - register_mime(mime) m = typeof(mime) if m <: MIME @eval mimestringtype(::$m) = JSONMIMEString @@ -96,6 +119,7 @@ for mime in [[MIME("application/vnd.vegalite.v2+json"), MIME("application/vnd.ve @eval _istextmime(::$m) = true end end + register_mime(mime) end display_mimestring(::JSONMIMEString, m::MIME, x) = (m, JSON.JSONText(limitstringmime(m, x))) diff --git a/src/inline.jl b/src/inline.jl index 920f09f7..7fd214f8 100644 --- a/src/inline.jl +++ b/src/inline.jl @@ -1,6 +1,3 @@ -import Base: display, redisplay - -struct InlineDisplay <: AbstractDisplay end # need special handling for showing a string as a textmime # type, since in that case the string is assumed to be @@ -31,25 +28,6 @@ function limitstringmime(mime::MIME, x) return String(take!(buf)) end -for mime in ijulia_mime_types - @eval begin - function display(d::InlineDisplay, m::MIME{Symbol($mime)}, x) - send_ipython(publish[], - msg_pub(execute_msg, "display_data", - Dict( - "metadata" => metadata(x), # optional - "data" => Dict($mime=>display_mimestring(m, x)[2])))) - end - displayable(d::InlineDisplay, ::MIME{Symbol($mime)}) = true - end -end - -# deal with annoying application/x-latex == text/latex synonyms -display(d::InlineDisplay, m::MIME"application/x-latex", x) = display(d, MIME("text/latex"), limitstringmime(m, x)) - -# deal with annoying text/javascript == application/javascript synonyms -display(d::InlineDisplay, m::MIME"text/javascript", x) = display(d, MIME("application/javascript"), limitstringmime(m, x)) - # If the user explicitly calls display("foo/bar", x), we send # the display message, also sending text/plain for text data. displayable(d::InlineDisplay, M::MIME) = _istextmime(M) From 8a7da6c9fff6ce7c29c4e13976096aec35862654 Mon Sep 17 00:00:00 2001 From: cnliao Date: Fri, 12 Oct 2018 22:37:49 +0800 Subject: [PATCH 7/7] fix typo --- src/execute_request.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute_request.jl b/src/execute_request.jl index ec8e1488..762f40d2 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -80,7 +80,7 @@ display_mimestring(::RawMIMEString, m::MIME, x) = (m, limitstringmime(m, x)) display_mimestring(m::MIME, x) = display_mimestring(mimestringtype(m), m, x) function display_mimestring(mime_array::Vector{<:MIME}, x) for m in mime_array - if showable(mime_array, x) + if showable(m, x) return display_mimestring(m, x) end end