GAVO Blog: Virtual Observatory Matters

T · that Simbad processes

facility_name
facility
SELECT distinct detail_value
FROM rr.res_detail
WHERE detail_xpath='/facility'

dachs
limits ALL
dachs limits
dachs imp
-l
<meta name="productType">timeseries</meta>

<meta name="productType">cube</meta>

dachs limits //obscore

<metaName="dataSource">theory</metaName>

<metaName="dataSource">artificial</metaName>

dachs limits
dachs limits
-l
10{pc}
@
SELECT TOP 20
  ra@{rad}, dec@{rad}, pmra@{rad/hyr}, pmdec@{rad/hyr}
FROM gaia.dr3lite

<publish render="edition" sets="ivo_managed">
  <meta>
    accessURL: \internallink{\rdId/static/myfile.txt}
    accessURL.resultType: text/plain
  </meta>
</publish>

doc:StaticFile
vs:WebBrowser
dachs val -vc ALL
val -vc
<publish sets="ivo_managed"/>

publish/@service
<your server URL>/browse/<rd-id>
<publish sets="local"/>
type=url
a
anchorText
resType: document
mixin="//siap#pgs"
<mixin have_bandpass_id="True">//siap2#pgs</mixin>

<mixin preview="access_url || '?preview=True'"
  >//obscore#publishObscoreLike</mixin>

//siap2#setMeta
bandpassId
bandpass_id
bandpassLo
em_min
bandpassHi
em_max
dateObs
t_min
t_max
obs_publisher_did
dl
 <meta name="_associatedDatalinkService">
  <meta name="serviceId">dl</meta>
  <meta name="idColumn">obs_publisher_did</meta>
</meta>

//siap2#computePGS
//siap2#getBandFromFilter
//siap#setMeta
//siap2#setMeta
dachs imp
//siap2#humanInput
#protoInput
allowed="siap2.xml"
<dbCore queriedTable="main">
  <FEED source="//siap2#parameters"/>
</dbCore>

siap2.xml
siap.xml
POS="CIRCLE ..."
render="siap.xml"
render="siap2.xml"
dachs pub q
dachs start
SELECT COUNT(*)
    FROM ivoa.obscore
    JOIN ivoa.obs_radio
    USING (obs_publisher_did)

obs_publisher_did
ivoa.obscore
ivoa.obscore
CREATE OR REPLACE VIEW ivoa.obscore AS
 SELECT 'image'::text AS dataproduct_type,
    NULL::text AS dataproduct_subtype,
    2::smallint AS calib_level,
    'BGDS'::text AS obs_collection,
    ...
 FROM bgds.data
UNION ALL
  SELECT 'image'::text AS dataproduct_type,
    NULL::text AS dataproduct_subtype,
    3::smallint AS calib_level,
  ...
[and 42 further subqueries that are union-ed together]

==== access_estsize ====

  bgds.data                      accsize/1024
  danish.data                    accsize/1024
  dfbsspec.ssa                   accsize/1024
  plts.data                      accsize/1024
  emi.main                       access_estsize (bigint)
  rosat.images                   accsize/1024
  califadr3.cubes                10
  robott.data                    accsize/1024
  k2c9vst.timeseries             accsize/1024
  dasch.narrow_plates            access_estsize (bigint)
  onebigb.ssa                    accsize/1024
  [...]

==== access_format ====

  bgds.data                      mime (text)
  danish.data                    mime (text)
  dfbsspec.ssa                   mime (text)
  plts.data                      mime (text)
  emi.main                       access_format (text)
  rosat.images                   mime (text)
  califadr3.cubes                'application/x-votable+xml;content=datalink'
  [...]

==== calib_level ====

  bgds.data                      2
  danish.data                    2
  dfbsspec.ssa                   2
  plts.data                      1
  emi.main                       calib_level (smallint)
  rosat.images                   2
  califadr3.cubes                3
  [...]

select pg_typeof(accsize/1024) from bgds.data limit 1

pol_xel
em_res_power
dachs val
-c
--compare-db
dachs val -vc ALL

import subprocess

from gavo import api

with api.getWritableAdminConn() as conn:
        conn.execute("DROP VIEW IF EXISTS ivoa.obscore")
        conn.execute("DROP VIEW IF EXISTS dasch.plates")

        for table_name in ["emi.main", "dasch.narrow_plates"]:
                conn.execute(f"ALTER TABLE {table_name} ALTER t_xel TYPE integer")

        for table_name in [
                        "emi.main", "dasch.narrow_plates", "ppakm31.cubes", "applause.main",]:
                conn.execute(f"ALTER TABLE {table_name} ALTER s_xel1 TYPE integer")
                conn.execute(f"ALTER TABLE {table_name} ALTER s_xel2 TYPE integer")

        for table_name in [
                        "emi.main",
                        "dasch.narrow_plates"]:
                conn.execute(f"ALTER TABLE {table_name} ALTER pol_xel TYPE integer")

        for table_name in [
                        "emi.main",
                        "dasch.narrow_plates",
                        "califadr3.cubes"]:
                conn.execute(f"ALTER TABLE {table_name} ALTER em_xel TYPE integer")
                conn.execute(f"ALTER TABLE {table_name} ALTER em_res_power TYPE real")

        for table_name in [
                        "emi.main",
                        "dasch.narrow_plates",
                        "ppakm31.cubes",
                        "applause.main",
                        "califadr3.cubes"]:
                conn.execute(f"ALTER TABLE {table_name} ALTER em_min TYPE real")
                conn.execute(f"ALTER TABLE {table_name} ALTER em_max TYPE real")

        for table_name in [
                        "emi.main",
                        "dasch.narrow_plates",
                        "applause.main"]:
                conn.execute(f"ALTER TABLE {table_name} ALTER s_resolution TYPE real")
                conn.execute(f"ALTER TABLE {table_name} ALTER s_pixel_scale TYPE real")
                conn.execute(f"ALTER TABLE {table_name} ALTER s_fov TYPE real")


for rd_id in ["emi/q", "califa/q3", "rome/q", "dasch/q", "ppakm31/q"]:
        subprocess.call(["dachs", "imp", "-m", rd_id])

subprocess.call(["dachs", "imp", "dasch/q", "make-view"])
subprocess.call(["dachs", "imp", "//obscore"])

dachs val -vc
select count(*)
from ivoa.obscore
natural join ivoa.obs_radio

SELECT count(*) FROM sometable;

select dataproduct_type, access_url, t_min, t_max
from ivoa.obscore
natural join ivoa.obs_radio
where t_min between 56000 and 56005

materialiseObscore: True

[ivoa]
/etc/gavo.rc
dachs imp //obscore
dachs limits //obscore

count(*)
ivoa._obscoresources
dachs adm dump //obscore
~/.ssh/known_hosts
sidedoor -R "*:80:dc.zah.uni-heidelberg.de:80" -R "*:443:dc.zah.uni-heidelberg.de:443" root@uhd-kruecke

#!/bin/sh
WATCH_URL=https://dc.g-vo.org/.well-known/security.txt
while true; do
        curl -s -S --connect-timeout 10 "$WATCH_URL" > /dev/null
        if [ $? -eq 28 ]; then
                # it's a timeout
                logger "Found dc.g-vo.org hanging, USR1-ing all sidedoors"
                killall -USR1 sidedoor
                date
        fi
        sleep 120
done

dachs start timeseries
<macDef raw="True" name="todo"><![CDATA[
]]></macDef>
raw="True"
macDef
q.rd
dachs limits
<coverage>
  <spectral>100[kHz] 21.5[cm]</spectral>
  <spectral>2[THz] 1[um]</spectral>
  <spectral>653[nm] 660[nm]</spectral>
  <spectral>912[Angstrom] 10[eV]</spectral>
  <spectral>20[GeV] 100[GeV]</spectral>
</coverage>

<column
<FEED source="//procs#witherror
err_name
<FEED source="//procs#witherror
  name="rv" type="double precision"
  unit="km/s" ucd="spect.dopplerVeloc"
  tablehead="RV_S"
  description="Radial velocity derived by the Serval pipeline"
  verbLevel="1"/>

values
dachs
limits
<LOOP>
  <csvItems>
  item, ucd
  u, U
  g, V
  r, R
  i, I
  z, I
  </csvItems>
  <events passivate="True">
    <FEED source="//procs#witherror name="mag_\item"
      unit="mag" ucd="phot.mag;em.opt.\ucd">
      tablehead="m_\item"
      description="Magnitude in \item band"/>
  </events>
</LOOP>

passivate="True"
dachs
test
dachs val -x //tap
macDef
regTest
\if
\if{\relpath}{http://example.edu/p/\relpath}
dachs
limits
ivoa.obscore
dataModel
<VOTABLE><RESOURCE><TABLE>
<FIELD name="objname" datatype="char" arraysize="*"/>
<DATA><TABLEDATA><TR>
<TD>Joachim Wambsganß</TD>
</TR></TABLEDATA></DATA></TABLE></RESOURCE></VOTABLE>

pos_ann = None
for desired_type in ["stc3:Coords", "stc2:Coords"]:
  for ann in ann.get_annotations(desired_type):
    pos_ann = ann

    if pos_ann is not None:
      break

if pos_ann is None:
  raise Exception("Don't understand any target annotation")

destruction
curl http://dc.g-vo.org/tap/user_tables/my_upload/destruction

curl -F DESTRUCTION=2038-01-19T03:14:07Z http://dc.g-vo.org/tap/user_tables/my_upload/destruction

destruction
INDEX
index
my_upload
curl -L -F INDEX=Kmag http://dc.g-vo.org/tap/user_tables/my_upload/index

-L
index
$ curl http://dc.g-vo.org/tap/user_tables/my_upload/index
Indexes on table tap_user.my_upload

CREATE INDEX my_upload_Kmag ON tap_user.my_upload (Kmag)

tmp.vot
_RAJ2000
_DEJ2000
pos.eq
curl -L -F INDEX=_RAJ2000 -F INDEX=_DEJ2000 \
  http://dc.g-vo.org/tap/user_tables/my_upload/index > /dev/null

Indexes on table tap_user.my_upload

CREATE INDEX my_upload__RAJ2000__DEJ2000 ON tap_user.my_upload (q3c_ang2ipix("_RAJ2000","_DEJ2000"))
CLUSTER my_upload__RAJ2000__DEJ2000 ON tap_user.my_upload
CREATE INDEX my_upload_Kmag ON tap_user.my_upload (Kmag)

CREATE TABLE tap_user.somepoints AS
SELECT TOP 30 preview, ssa_location
FROM gdr3spec.ssameta

curl -LF INDEX=ssa_location http://dc.g-vo.org/tap/user_tables/somepoints/index

Indexes on table tap_user.somepoints

CREATE INDEX ndpmaliptmpa_ssa_location ON tap_user.ndpmaliptmpa USING GIST (ssa_location)
CLUSTER ndpmaliptmpa_ssa_location ON tap_user.ndpmaliptmpa

USING GIST
curl -L -F INDEX=_Glon -F INDEX=_DEJ2000 \
  http://dc.g-vo.org/tap/user_tables/my_upload/index > /dev/null

<vocabulary
uri>#<term identifier>
>>> from pyvo import registry
>>> print(registry.search(registry.UAT("variable-stars")))
<DALResultsTable length=2010>
              ivoid               ...
                                  ...
              object              ...
--------------------------------- ...
         ivo://cds.vizier/b/corot ...
          ivo://cds.vizier/b/gcvs ...
           ivo://cds.vizier/b/vsx ...
          ivo://cds.vizier/i/280b ...
           ivo://cds.vizier/i/345 ...
           ivo://cds.vizier/i/350 ...
                              ... ...
            ivo://cds.vizier/v/97 ...
         ivo://cds.vizier/vii/293 ...
   ivo://org.gavo.dc/apass/q/cone ...
ivo://org.gavo.dc/bgds/l/meanphot ...
     ivo://org.gavo.dc/bgds/l/ssa ...
     ivo://org.gavo.dc/bgds/q/sia ...

res_subject
>>> print(registry.search(registry.UAT("varialbe-stars")))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/msdemlei/gavo/src/pyvo/pyvo/registry/rtcons.py", line 713, in __init__
    raise dalq.DALQueryError(
pyvo.dal.exceptions.DALQueryError: varialbe-stars does not identify an IVOA uat concept (see http://www.ivoa.net/rdf/uat).

expand_up
expand_down
>>> print(registry.search(registry.UAT("supernovae", expand_down=10)))
<DALResultsTable length=593>
                 ivoid                   ...
                                         ...
                 object                  ...
---------------------------------------- ...
                   ivo://cds.vizier/b/sn ...
                 ivo://cds.vizier/ii/159 ...
                 ivo://cds.vizier/ii/189 ...
                 ivo://cds.vizier/ii/205 ...
                ivo://cds.vizier/ii/214a ...
                 ivo://cds.vizier/ii/218 ...
                                     ... ...
           ivo://cds.vizier/j/pasp/122/1 ...
       ivo://cds.vizier/j/pasp/131/a4002 ...
           ivo://cds.vizier/j/pazh/30/37 ...
          ivo://cds.vizier/j/pazh/37/837 ...
ivo://edu.gavo.org/eurovo/aida_snconfirm ...
                ivo://mast.stsci/candels ...

>>> print(registry.get_RegTAP_query(registry.UAT("supernovae", expand_down=10)))
SELECT
  [crazy stuff elided]
WHERE
(ivoid IN (SELECT DISTINCT ivoid FROM rr.res_subject WHERE res_subject in (
  'core-collapse-supernovae', 'hypernovae', 'supernovae',
  'type-ia-supernovae', 'type-ib-supernovae', 'type-ic-supernovae',
  'type-ii-supernovae')))
GROUP BY [whatever]

gavo_vocmatch(vocname STRING, term STRING, matchagainst STRING) -> INTEGER

SELECT ivoid FROM rr.res_subject
WHERE 1=gavo_vocmatch('uat', 'supernovae', res_subject)

registry.UAT("supernovae", expand_down=1)
gavo_vocmatch
gavo_vocmatch
gavo_vocmatch
class UAT(SubqueriedConstraint):
    _keyword = "uat"
    _subquery_table = "rr.res_subject"
    _condition = "res_subject in {query_terms}"
    _uat = None

    @classmethod
    def _expand(cls, term, level, direction):
        result = {term}
        new_concepts = cls._uat[term][direction]
        if level:
            for concept in new_concepts:
                result |= cls._expand(concept, level-1, direction)
        return result

    def __init__(self, uat_keyword, *, expand_up=0, expand_down=0):
        if self.__class__._uat is None:
            self.__class__._uat = vocabularies.get_vocabulary("uat")["terms"]

        if uat_keyword not in self._uat:
            raise dalq.DALQueryError(
                f"{uat_keyword} does not identify an IVOA uat"
                " concept (see http://www.ivoa.net/rdf/uat).")

        query_terms = {uat_keyword}
        if expand_up:
            query_terms |= self._expand(uat_keyword, expand_up, "wider")
        if expand_down:
            query_terms |= self._expand(uat_keyword, expand_down, "narrower")

        self._fillers = {"query_terms": query_terms}

_subquery_table
_condition
str.format
_fillers
_condition
make_SQL_literal
_keyword
registry.search(uat="foo.bar")
registry.search(registry.UAT("foo.bar"))
__class__
_expand
direction
wider
narrower
level
maxrec
run_sync
TAPService.run_sync
run_async
run_sync
execution_duration
main
import time
import pyvo

QUERY = """
SELECT
    source_id, phot_g_mean_mag, pseudocolour,
    pseudocolour_error, phot_g_mean_flux_over_error
FROM gedr3dist.litewithdist
WHERE
    r_med_photogeo between 290 and 310
    AND ruwe<1.4
    AND pseudocolour BETWEEN 1.0 AND 1.8
"""

def send_table_url_to_topcat(conn, table_url):
    client_id = pyvo.samp.find_client_id(conn, "topcat")
    message = {
        "samp.mtype": "table.load.votable",
        "samp.params": {
            "url": table_url,
            "name": "TAP result",}
    }
    conn.notify(client_id, message)


def main():
    svc = pyvo.dal.TAPService("http://dc.g-vo.org/tap")
    job = svc.submit_job(QUERY, maxrec=3000)
    try:
        job.execution_duration=10000  # that's 10000 seconds
        job.run()
        job.wait()
        assert job.phase=="COMPLETED"

        with pyvo.samp.connection(addr="127.0.0.1") as conn:
            send_table_url_to_topcat(conn, job.result_uri)
    finally:
        job.delete()

if __name__=="__main__":
    main()

maxrec=3000
execution_duration
maxrec=1000000
get_contact
>>> pyvo.registry.search(ivoid="ivo://org.gavo.dc/tap")[0].get_contact()
'GAVO Data Centre Team (+49 6221 54 1837) <gavo@ari.uni-heidelberg.de>'

source_id
SELECT max(source_id) FROM gaia.dr3lite

N_PART
import pyvo

MAX_ID, N_PART = 6917528997577384320+1, 100
partition_limits = [(MAX_ID//N_PART)*i
  for i in range(N_PART+1)]

svc = pyvo.dal.TAPService("http://dc.g-vo.org/tap")
main_query = "SELECT count(*) FROM ({part}) AS q"

for lower, upper in zip(partition_limits[:-1], partition_limits[1:]):
  result = svc.run_sync(main_query.format(part=
    "SELECT * FROM gaia.dr3lite"
    "  WHERE source_id BETWEEN {} and {} ".format(lower, upper-1)))
  print(result)

MAX_ID
range
from astropy import table
import pyvo

MAX_ID, ROW_TARGET = 6917528997577384320+1, 10000000

ENDPOINT = "http://dc.g-vo.org/tap"

# the 10000 is just the number of bins to use; make it too small, and
# your inital bins may already overflow ROW_TARGET
ID_DIVISOR = MAX_ID/10000

DISTRIBUTION_QUERY = f"""
select round(source_id/{ID_DIVISOR}) as bin, count(*) as ct
from gaia.dr3lite
group by bin
"""


def get_bin_sizes():
  """returns a ordered sequence of (bin_center, num_objects) rows.
  """
  # since the partitioning query already is expensive, cache it,
  # and use the cache if it's there.
  try:
    with open("partitions.vot", "rb") as f:
      tbl = table.Table.read(f)
  except IOError:
    # Fetch from source; takes about 1 hour
    print("Fetching partitions from source; this will take a while"
      " (provide partitions.vot to avoid re-querying)")
    svc = pyvo.dal.TAPService(ENDPOINT)
    res = svc.run_async(DISTRIBUTION_QUERY, maxrec=1000000)
    tbl = res.table
    with open("partitions.vot", "wb") as f:
      tbl.write(output=f, format="votable")

  res = [(row["bin"], row["ct"]) for row in tbl]
  res.sort()
  return res


def get_partition_limits(bin_sizes):
  """returns a list of limits of source_id ranges exhausting the whole
  catalogue.

  bin_sizes is what get_bin_sizes returns (and it must be sorted by
  bin center).
  """
  limits, cur_count = [0], 0
  for bin_center, bin_count in bin_sizes:
    if cur_count+bin_count>MAX_ROWS:
      limits.append(int(bin_center*ID_DIVISOR-ID_DIVISOR/2))
      cur_count = 0
    cur_count += bin_count
  limits.append(MAX_ID)
  return limits


def get_data_for(svc, query, low, high):
  """returns a TAP result for the (simple) query in the partition
  between low and high.

  query needs to query the ``sample`` table.
  """
  job = svc.submit_job("WITH sample AS "
    "(SELECT * FROM gaia.dr3lite"
    "  WHERE source_id BETWEEN {} and {}) ".format(lower, upper-1)
    +query, maxrec=ROW_TARGET)
  try:
    job.run()
    job.wait()
    return job.fetch_result()
  finally:
    job.delete()


def main():
  svc = pyvo.dal.TAPService(ENDPOINT)
  limits = get_partition_limits(get_bin_sizes())
  for ct, (low, high) in enumerate(zip(limits[:-1], limits[1:])):
    print("{}/{}".format(ct, len(limits)))
    res = get_data_for(svc, <a query over a table sample>, low, high-1)
    # do your thing here

dachs imp //tap_user

dachs adm adduser
<execute title="harvest RofR" every="40000">
  <job><code>
      execDef.spawnPython("bin/harvestRofR.py")
  </code></job>
</execute>

<execute on="loaded" title="define functions"><job>
  <setup imports="h5py, numpy"/>
  <code>
  def get_photpoints(field, quadrant, quadrant_id):
    """returns the photometry points for the specified time series
    from the HDF5 as a numpy array.

    [...]
    """
    dest_path = "data/ROME-FIELD-{:02d}_quad{:d}_photometry.hdf5".format(
      field, quadrant)
    srchdf = h5py.File(rd.getAbsPath(dest_path))
    _, arr = next(iter(srchdf.items()))

    photpoints = arr[quadrant_id-1]
    photpoints = numpy.array(photpoints)
    photpoints[photpoints==0] = numpy.nan
    photpoints[photpoints==-9999.99] = numpy.nan

    return photpoints


  def get_photpoints_for_rome_id(rome_id):
    """as get_photpoints, but taking an integer rome_id.
    """
    field = rome_id//10000000
    quadrant = (rome_id//1000000)%10
    quadrant_id = (rome_id%1000000)
    base.ui.notifyInfo(f"{field} {quadrant} {quadrant_id}")
    return get_photpoints(field, quadrant, quadrant_id)

  rd.get_photpoints = get_photpoints
  rd.get_photpoints_for_rome_id = get_photpoints_for_rome_id
</code></job></execute>

dachs start
$ dachs start list
epntap -- Solar system data via EPN-TAP 2.0
siap -- Image collections via SIAP2 and TAP
scs -- Catalogs via SCS and TAP
ssap+datalink -- Spectra via SSAP and TAP, going through datalink
taptable -- Any sort of data via a plain TAP table

taptable
application/x-votable+xml;version=1.5
application/x-votable+xml;version=1.6
dachs pub //adql
dachs limits
%
dachs
pub
meta.bib.bibcode
apt upgrade
stilts taplint
user_tables
user_tables
$ curl -o tmp.vot https://docs.g-vo.org/upload_for_regressiontest.vot
$ curl -H "content-type: application/x-votable+xml" -T tmp.vot \
  http://dc.g-vo.org/tap/user_tables/my_upload
Query this table as tap_user.my_upload

$ curl -H "content-type: application/x-votable+xml" -T tmp.vot
  http://dc.g-vo.org/tap/user_tables/join
... <INFO name="QUERY_STATUS" value="ERROR">'join' cannot be used as an
  upload table name (which must be regular ADQL identifiers, in
  particular not ADQL reserved words).</INFO> ...

tap_user.my_upload
$ curl http://dc.g-vo.org/tap/user_tables/my_upload | xmlstarlet fo
<vtm:table [...]>
  <name>tap_user.my_upload</name>
  <column>
    <name>"_r"</name>
    <description>Distance from center (RAJ2000=274.465528, DEJ2000=-15.903352)</description>
    <unit>arcmin</unit>
    <ucd>pos.angDistance</ucd>
    <dataType xsi:type="vs:VOTableType">float</dataType>
    <flag>nullable</flag>
  </column>
  ...

user_tables
$ curl -X DELETE http://dc.g-vo.org/tap/user_tables/my_upload
Dropped user table my_upload

uptest
  $ curl -H "content-type: application/x-votable+xml" -T tmp.vot \
  --user uptest:uptest \
  http://dc.g-vo.org/tap/user_tables/privtab
Query this table as tap_user.privtab

$ curl --user uptest:uptest http://dc.g-vo.org/tap/user_tables | xmlstarlet fo
<vtm:tableset ...>
  <schema>
    <name>tap_user</name>
    <description>A schema containing users' uploads. ...  </description>
    <table>
      <name>tap_user.privtab</name>
      <column>
        <name>"_r"</name>
        <description>Distance from center (RAJ2000=274.465528, DEJ2000=-15.903352)</description>
        <unit>arcmin</unit>
        <ucd>pos.angDistance</ucd>
        <dataType xsi:type="vs:VOTableType">float</dataType>
        <flag>nullable</flag>
      </column>
      ...
    </table>
    <table>
      <name>tap_user.my_upload</name>
      <column>
        <name>"_r"</name>
        <description>Distance from center (RAJ2000=274.465528, DEJ2000=-15.903352)</description>
        <unit>arcmin</unit>
        <ucd>pos.angDistance</ucd>
        <dataType xsi:type="vs:VOTableType">float</dataType>
        <flag>nullable</flag>
      </column>
      ...
    </table>
  </schema>
</vtm:tableset>

destruction
dachs imp //tap_user

dachs
drop --system //tap_user
make
\includegraphics[height=5cm]{vo-qr.png}}

fluggi.pdf: fluggi.tex vo-qr.png <and possibly more images>

vo-qr.png
vo.url
-qr.png
https://doi.org/10.21938/avVAxDlGOiu0Byv7NOZCsQ

%-qr.png: %.url
        python qrmake.py $<

qrmake.py
import sys
import qrcode

with open(sys.argv[1], "rb") as f:
        content = f.read().strip()
output_code = qrcode.QRCode(border=0)
output_code.add_data(content)

dest_name = sys.argv[1].replace(".url", "")+"-qr.png"
output_code.make_image().save(dest_name)

productTypeServed
<meta name="productTypeServed">timeseries</meta>

obscore-extraevents
dachs limits //obscore
dachs val
dachs pub //obscore
dachs
limits //obscore
gaia_source
gaia_source
<resource schema="dfbsspec">
  <meta name="schema-rank">100</meta>
    ...
    <table id="spectra" onDisk="True" adql="True">
      <meta name="table-rank">1</meta>

dachs pub
-k
dachs pub -k ALL

ExternalProcLinkDef
inputKeys
<metaMaker>
  <code>
    footprint = descriptor.skyWCS.calcFootprint(descriptor.hdr)
    ra_range = MS(Values,
      min=min(footprint[:,0]),
      max=max(footprint[:,0]))
    dec_range = MS(Values,
      min=min(footprint[:,1]),
      max=max(footprint[:,1]))

    yield ExternalProcLinkDef(
      descriptor.pubDID, [
        MS(InputKey, name="DATASET_ID", type="text",
          ucd="meta.id;meta.main",
          description="Dataset to operate on",
          content_=descriptor.pubDID),
        MS(InputKey, name="RA_MIN",
          unit="deg", ucd="pos.eq.ra;stat.min",
          values=ra_range),
        MS(InputKey, name="RA_MAX",
          unit="deg", ucd="pos.eq.ra;stat.max",
          values=ra_range),
        MS(InputKey, name="DEC_MIN",
          unit="deg", ucd="pos.eq.dec;stat.min",
          values=dec_range),
        MS(InputKey, name="DEC_MAX",
          unit="deg", ucd="pos.eq.dec;stat.max",
          values=dec_range)],
      "http://example.org/cgi-bin/cutout.pl",
      "Cutout",
      "External service doing a cutout on this dataset")
  </code>
</metaMaker>

dir / name
os.path.join(dir, name)
base.getConfig("inputsDir")
sourceToken
self.sourceToken[:-5]
>>> n = pathlib.Path("/a/b/c.fits")
>>> n[:-5]
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: 'PosixPath' object is not subscriptable

getConfig
>>> str(n)[:-5]
'/a/b/c'

classify
http://<your-hostname>/.well-known/security.txt
/usr/lib/python3/dist-packages/gavo/resources/templates/
dachs serve start
dachs serve stop
/getproduct?key=some/accref
/getproduct/some/accref
self.location
<iterator><code>
  with open(self.sourceToken) as f:
    for line_no, line in enumerate(f):
      self.location = f"{self.sourceToken}, {line_no}"
      # not do whatever you need to do on line
</code></iterator>

addExtras(descriptor)
setup
<descriptorGenerator procDef="//soda#fromStandardPubDID">
  <bind name="accrefPrefix">"dasch/q/"</bind>
  <bind name="contentQualifier">"image"</bind>
  <setup>
    <code>
      def addExtras(descriptor):
        descriptor.suppressAutoLinks = True
        with base.getTableConn() as conn:
          descriptor.extMeta = next(conn.queryToDicts(
            "SELECT * FROM dasch.plates"
            " WHERE obs_publisher_did = %(did)s",
            {"did": descriptor.pubDID}))
    </code>
  </setup>
</descriptorGenerator>

apt upgrade
pyvo/docs/dal > watch-doctests.py index.rst | head -30
---0.00---------------

import pyvo as vo
---0.94---------------

service = vo.dal.SIAService("http://dc.zah.uni-heidelberg.de/lswscans/res/positions/siap/siap.xml")
---0.94---------------

print(service.description)
Scans of plates kept at Landessternwarte Heidelberg-Königstuhl. They
were obtained at location, at the German-Spanish Astronomical Center
(Calar Alto Observatory), Spain, and at La Silla, Chile. The plates
cover a time span between 1880 and 1999.

Specifically, HDAP is essentially complete for the plates taken with
the Bruce telescope, the Walz reflector, and Wolf's Doppelastrograph
at both the original location in Heidelberg and its later home on
Königstuhl.
---1.02---------------

import pyvo as vo
---1.02---------------

from astropy.coordinates import SkyCoord
---1.02---------------

from astropy.units import Quantity

dasch.narrow_plates
dasch.wide_plates
s_region
SELECT instrument_name, avg(area(s_region)) as meanarea
FROM dasch.plates
GROUP BY instrument_name

SELECT ROUND(AREA(s_region)/100)*100 AS platebin, count(*) AS ct
FROM dasch.plates
GROUP BY platebin

s_xel1
s_xel2
import pyvo

pos = (202.46, 47.19)

svc = pyvo.dal.TAPService("http://dc.g-vo.org/tap")
res = svc.run_sync(f"""
    SELECT *
    FROM
        dasch.narrow_plates
    WHERE
        distance(s_ra, s_dec, {pos[0]}, {pos[1]})<0.5""")
for rec in res:
    prod = rec.processed(circle=pos+(0.1,))
    dest_name = rec["dasch_id"]+".cutout.fits"
    print(dest_name)
    with open(dest_name, "wb") as f:
        f.write(prod.data)

from astropy import table
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.time import Time

ephem = table.Table.read(
  "https://vo.imcce.fr/webservices/miriade/ephemcc.php?-from=vespa"
  "&-name=c:p/encke&-ep=1941-04-01&-nbd=90&-step=1d&-observer=500"
  &-mime=votable")

parsed = SkyCoord(ephem["ra"], ephem["dec"], unit=(u.hourangle, u.deg))
ephem["ra"] = parsed.ra.degree
ephem["dec"] = parsed.dec.degree

parsed = Time(ephem["epoch"])
ephem["epoch"] = parsed.mjd

svc = pyvo.dal.TAPService("http://dc.g-vo.org/tap")
res = svc.run_sync("""
    SELECT *
    FROM
        dasch.plates
        JOIN tap_upload.orbit
        ON (1=CONTAINS(POINT(ra, dec), s_region))
    WHERE
        t_min<epoch
        AND t_max>epoch""",
    uploads={"orbit": ephem})

dasch.narrow_plates
dasch.plates
s_region
iter_datalinks
with pyvo.samp.connection() as conn:
    for dl in res.iter_datalinks():
        link = next(dl.bysemantics("#preview-image"))
        pyvo.samp.send_image_to(
            conn,
            link.access_url,
            client_name="Aladin")

pyvo.samp
from pyvo import discover
from astropy import units as u
from astropy import time

datasets, log = discover.images_globally(
  space=(339.49, 3.1, 0.1),
  spectrum=650*u.nm,
  time=(time.Time('1995-01-01'), time.Time('1995-12-31')))

datasets
origin_service
ivo:
https://dc.g-vo.org/I/
https://dc.g-vo.org/LP/
Skipping ivo://org.gavo.dc/lswscans/res/positions/siap because it is served by ivo://org.gavo.dc/__system__/obscore/obscore
Skipping ivo://org.gavo.dc/rosat/q/im because it is served by ivo://org.gavo.dc/__system__/obscore/obscore
Obscore GAVO Data Center Obscore Table: 2 records
SIA2 The VO @ ASTRON SIAP Version 2 Service: 0 records
SIA2 ivo://au.csiro/casda/sia2 skipped: ReadTimeout: HTTPSConnectionPool(host=&apos;casda.csiro.au&apos;, port=443): Read timed out. (read timeout=20)
SIA2 CADC Image Search (SIA): 0 records
SIA2 European HST Archive SIAP service: 0 records
...

from pyvo import discover
from pyvo import registry
from astropy import units as u
from astropy import time

def say(discoverer, s):
        print(s)

datasets, log = discover.images_globally(
  space=(274.6880, -13.7920, 1),
  time=(time.Time('1995-01-01'), time.Time('1995-12-31')),
  services=registry.search(registry.Datamodel("obscore")),
  watcher=say)

watcher
discover.images_globally
discover.images_globally
discovery_log
Skipping ivo://org.gavo.dc/lswscans/res/positions/siap because it is served by ivo://org.gavo.dc/__system__/obscore/obscore
Skipping ivo://org.gavo.dc/rosat/q/im because it is served by ivo://org.gavo.dc/__system__/obscore/obscore

select ivoid, related_id from
rr.relationship
natural join rr.capability
where
standard_id like 'ivo://ivoa.net/std/sia%'
and relationship_type='isservedby'

s_region
ImageFound
"(distance(s_ra, s_dec, {}, {}) < {}".format(
  self.center[0], self.center[1], self.radius)
+" or 1=intersects(circle({}, {}, {}), s_region))".format(
  self.center[0], self.center[1], self.radius))

s_ra
s_dec
s_fov
s_region
s_region
s_ra
s_dec
s_fov
s_region
1=intersects(circle(.), s_region)
s_region
s_ra
s_dec
s_fov
distance(.) < {self.radius}+s_fov
s_fov
s_ra
s_dec
s_fov
s_region
virtualenv --system-site-packages global-datasets
. global-datasets/bin/activate
cd global-datasets
git clone https://github.com/msdemlei/pyvo
cd pyvo
git checkout global-datasets
pip install .

/get
Cites: Some external thing
Cites.altIdentifier: doi:10.fake/123412349876

<meta>
type="json"
type="jsonb"
<coverage fallbackTo="__system__/obscore"/>

dachs limits //obscore
getFirstVOTableRow
rejectExtras=False
arr_count
ivo_
gavo_
gavo_
+
apt update && apt upgrade
SELECT * FROM…
apt install astrometry.net
apt search astrometry-data
astrometry-data-tycho2-10-19
astrometry-data-tycho2-09
astrometry-data-tycho2-08
astrometry-data-tycho2-07
sp_indices
from gavo import api

class MyObsCalibrator(api.AnetHeaderProcessor):
  indexPath = "/usr/share/astrometry"
  sp_indices = ["index-tycho2-09*.fits",
    "index-tycho2-10*.fits",
    "index-tycho2-11*.fits",]

"""
Create an index for astrometry.net and a few small fields based on Gaia.

Be sure to adapt this for your use case; for instance, if what your are
calibrating will be from only a part of the sky, pick specific healpixes
(perhaps on a different level; below, we're using level 5).  Also consider
changing the target epoch, the photometry, or the magnitude limit.

This script takes the sample positions from a text file; have
space-separated pairs of ra and dec in targets.txt.
"""

import os
import subprocess

from astropy.table import Table
import pyvo

# 0 is for images of about two arcminutes, 10 for about degree, 12 for two
# degrees, etc.
SIZE_PRESET = 1

# The typical radius of your images in degrees (this is the size of our cone
# searches, so cut some slack); this needs to be changed in unison with
# SIZE_PRESET
IMAGE_RADIUS = 1/10.


def get_target_table():
    """must return an astropy table with columns ra and dec in degrees.

    (of course, if you have your data in a proper format with actual metadata,
    you don't need any of the ugly magic).
    """
    targets = Table.read("targets.txt", format="ascii")
    targets["col1"].name, targets["col2"].name = "ra", "dec"
    targets["ra"].meta = {"ucd": "pos.eq.ra;meta.main"}
    targets["dec"].meta = {"ucd": "pos.eq.dec;meta.main"}
    return targets


def main():
    tap_service = pyvo.dal.TAPService("http://dc.g-vo.org/tap")
    res = tap_service.run_async(f"""
        SELECT g.ra as RA, g.dec as DEC, phot_g_mean_mag as MAG
        FROM gaia.dr3lite AS g
        JOIN TAP_UPLOAD.t1 as mine
            ON DISTANCE(mine.ra, mine.dec, g.ra, g.dec)<{IMAGE_RADIUS}""",
      uploads={"t1": get_target_table()})

    cat_file = "basic-cat.fits"
    res.to_table().write(cat_file, format="fits", overwrite=True)

    try:
        subprocess.run(["build-astrometry-index", "-i", cat_file,
            "-o", f"./index-custom-{SIZE_PRESET:02d}.fits",
            "-P", str(SIZE_PRESET), "-S", "MAG"])
    finally:
        os.unlink(cat_file)


if __name__=="__main__":
    main()

index-custom-01.fits
basic-cat.fits
custom-index
from gavo import api

RD = api.getRD("myres/q")

class MyObsCalibrator(api.AnetHeaderProcessor):
  indexPath = RD.resdir
  sp_indices = ["custom-index/index-custom-01.fits"]

"""
Create a full-sky index for bright stars and astrometry.net based on Gaia.

This only works for rather bright stars because the Gaia service will refuse
to server more than ~1e7 objects.

Make sure to choose SIZE_PRESET to your use case (19 means 30 deg,
10 about a degree, two preset steps are about a factor two in scale).
"""

import os
import subprocess

import pyvo

# see the module docstring
SIZE_PRESET = 12

# ignore stars fainter than this; you can't go below 14 all-sky with Gaia
# and the GAVO DC server
MAX_MAG = 12

# Epoch to transform the stars to
TARGET_EPOCH = 1910


def main():
    tap_service = pyvo.dal.TAPService("http://dc.g-vo.org/tap")
    res = tap_service.run_async(f"""
        SELECT pos[1] as RA, pos[2] as DEC, mag as MAG
        FROM (
            SELECT phot_bp_mean_mag AS mag,
                ivo_epoch_prop(ra, dec, parallax,
                    pmra, pmdec, radial_velocity, 2016, {TARGET_EPOCH}) as pos
            FROM gaia.dr3lite
          WHERE phot_bp_mean_mag<{MAX_MAG}) AS q""")

    cat_file = "current.fits"
    res.to_table().write(cat_file, format="fits", overwrite=True)

    try:
        for size_preset in range(SIZE_PRESET-1, SIZE_PRESET+2):
            subprocess.run(["build-astrometry-index", "-i", cat_file,
                "-o", f"./index-custom-{size_preset:02d}.fits",
                "-P", str(size_preset), "-S", "MAG"])
    finally:
        os.unlink(cat_file)


if __name__=="__main__":
    main()

custom-index
from gavo import api

RD = api.getRD("myres/q")

class MyObsCalibrator(api.AnetHeaderProcessor):
  indexPath = RD.resdir
  sp_indices = ["custom-index/index-custom-*.fits"]

dachs
start
spectralAxis
velocityAxis
axisMetaOverrides
<unionGrammar>
  <handles pattern=".*\.txt$">
    <reGrammar...>
  </handles>
  <handles pattern=".*\.csv$">
    <csvGrammar...>
  </handles>
</unionGrammar>

/var/gavo/etc
from gavo import api
<setup
imports="gavo.api"/>
api.local.<name>
local.py
def parse_babylonian_time(raw_time:str) -> float:
  """Tries to interpret raw_time as a time in one of the many forms
  our observers like so much.

  Here is the syntaxes supported by the function:

  >>> parse_babylonian_time("1h")
  3600.0
  >>> parse_babylonian_time("4h30m")
  16200.0
  >>> parse_babylonian_time("1h30m20s")
  5420.0
  >>> parse_babylonian_time("20m")
  1200.0
  >>> parse_babylonian_time("10.5m")
  630.0
  >>> parse_babylonian_time("1m10s")
  70.0
  >>> parse_babylonian_time("15s")
  15.0
  >>> parse_babylonian_time("s23m")
  Traceback (most recent call last):
  ValueError: Cannot understand time 's23m'
  """
  mat = re.match(
    r"^(?P<hours>\d+(?:\.\d+)?h)?"
    r"(?P<minutes>\d+(?:\.\d+)?m)?"
    r"(?P<seconds>\d+(?:\.\d+)?s)?$", raw_time)
  if mat is None:
    raise ValueError(f"Cannot understand time '{raw_time}'")
  parts = mat.groupdict()

  return (float((parts["hours"] or "0h")[:-1])*3600
    + float((parts["minutes"] or "0m")[:-1])*60
    + float((parts["seconds"] or "0s")[:-1]))

dachs test
local.py
votable:Coords
value
space
location
contentQualifier
descriptor.makeLink[FromFile]
ProcLinkDef
inputOptional
[db]poolSize
CONTAINS(CIRCLE(23, 42, 1),
some_moc)
servicetype='image'
<INFO property="doap:homepage"
  magic-attribute="magic-value"
  >http://foo.bar"</INFO>

<> doap:homepage <http://foo.bar>

SELECT * FROM rr.resource JOIN rr.stc_spectral WHERE
  1=ivo_interval_overlaps(spectral_start, spectral_end,
      ivo_specconv(658, 'nm', 'J'), ivo_specconv(654, 'nm', 'J'))

von der Heide, J.
J. von der Heide
LIKE '%von der
Heide%'
get_contact
ATLAS DR1, SIAP, Images
NOT PROVIDED
???
optical lines equivalent width catalog
6dF Data Release 3 Spectra
COROT
N2
TAP
CADC image/cube HiPS service
Galaxy:general
  </capability>
</ri:Resource>

?
??
???
RA
meta.code;phot.flux.density;arith.ratio;em.ir.15-30um;em.radio.750-1500mhz
ABmag/arcsec2
**
$ stilts calc 'vounitStatus("ABmag/arcsec2")'
  BAD_SYNTAX
$ stilts calc 'vounitStatus("mag/arcsec**2")'
  OK

>>> 3145//4  # ...within this HEALPix on level 5...
786
>>> 3145*4, (3145+1)*4  # ..and covers these on level 7...
(12580, 12584)

SELECT source_id/8796093022208 AS pix,
  AVG(phot_bp_mean_mag-phot_rp_mean_mag) AS avgcol
FROM gaia.edr3lite
WHERE distance(ra, dec, 246.7, -24.5)<2
GROUP by pix

pix
avgcol
gavo_simbadpoint
SELECT TOP 1 ivo_healpix_index(3, gavo_simbadpoint('M42'))
FROM tap_schema.tables

SELECT source_id/8796093022208 AS pix,
  AVG(phot_bp_mean_mag-phot_rp_mean_mag) AS avgcol
FROM gaia.edr3lite
WHERE source_id BETWEEN 334*9007199254740992 AND 335*9007199254740992-1
GROUP by pix

source_id BETWEEN 83*36028797018963968 AND 84*36028797018963968-1
OR source_id BETWEEN 86*36028797018963968 AND 87*36028797018963968-1

SELECT ROUND(r_med_photogeo/200)*200 AS distbin, COUNT(*) as n,
    AVG(phot_bp_mean_mag-phot_rp_mean_mag) AS avgcol
FROM gaia.dr3lite
JOIN gedr3dist.main USING (source_id)
WHERE source_id BETWEEN 88974*35184372088832 and 88975*35184372088832-1
GROUP BY distbin

source_id/(POWER(2,
35)*POWER(4, 3)
my-image.fits
apt install
source-extractor
default.param
ALPHA_SKY
DELTA_SKY
X_IMAGE
Y_IMAGE
MAG_ISO
FLUX_AUTO
ELONGATION

DETECT_MINAREA
DETECT_THRESH
default.control
CATALOG_TYPE     FITS_1.0
CATALOG_NAME     img.axy
PARAMETERS_NAME  default.param
FILTER           N
DETECT_MINAREA   30
DETECT_THRESH    4
SEEING_FWHM      1.2

source-extractor -c default.control my-image.fits

img.axy
img.axy
pos.eq.ra
pos.eq.dec
ALPHA_SKY
DELTA_SKY
TOP 1000
SELECT
   source_id, ra, dec, phot_bp_mean_mag,
   tc.*
   FROM gaia.dr3lite AS db
   JOIN TAP_UPLOAD.t1 AS tc
   ON 1=CONTAINS(POINT('ICRS', db.ra, db.dec),
                 CIRCLE('ICRS', tc.ALPHA_SKY, tc.DELTA_SKY, 4./3600.))

phot_bp_mean_mag
MAG_ISO
MAG_ISO
ra-alpha_sky
dec-delta_sky
X_IMAGE
Y_IMAGE
/usr/share/doc/python3-gavo/changelog.gz
localSemantics
makeLink
makeLinkFromFile
GRANT ALL ON SCHEMA public TO
gavoadmin
apt update && apt upgrade
#aws-metadata
WHERE keyword matches
table1.descripition OR keyword matches table2.subject
SELECT main_id, ra, dec
FROM basic
WHERE otype='WR*'

SELECT main_id, ra, dec, id
FROM basic
  JOIN ident
  ON (oid=oidref)
WHERE otype='WR*'
  AND id LIKE 'Gaia DR3%'

id
gdr3spec.spectra
gdr3spec.withpos
WITH wrids AS (
        SELECT source_id, main_id
        FROM gaia.edr3lite AS l
    JOIN tap_upload.t1 AS u
        ON (DISTANCE(l.ra, l.dec, u.ra, u.dec)<0.001))
SELECT main_id, source_id, flux, flux_error
FROM gdr3spec.spectra
JOIN wrids USING (source_id)

tap_upload.t1
SELECT source_id, main_id
FROM gaia.edr3lite AS l
  JOIN tap_upload.t1 AS u
        ON (DISTANCE(l.ra, l.dec, u.ra, u.dec)<0.001)

WITH normalised AS (
SELECT source_id, main_id,
        flux/arr_sum(flux) as nflux, flux
FROM gdr3spec.spectra
JOIN tap_upload.t10
USING (source_id))
SELECT
  source_id, main_id,
  flux, nflux,
  arr_avg(nflux*nflux)-POWER(arr_avg(nflux),2) AS sd,
  nflux[19] as em
FROM normalised

arr_sum
arr_avg
nflux*nflux
nflux
nflux[19]
em
sd
SELECT summed/arr_sum(summed) AS tpl
FROM (
  SELECT SUM(flux) AS summed
  FROM gdr3spec.spectra
  JOIN tap_upload.t19
  USING (source_id)) AS q

t19
SELECT
SUM(flux)
summed/arr_sum(summed)
SUM(arr)
array_sum(arr)
SELECT TOP 200000
  source_id,
  arr_sum(
    arr_map(
      power(x,2),tpl-flux/arr_sum(flux))) AS dist2,
  arr_sum(flux_error/flux) AS errs
FROM gdr3spec.spectra, tap_upload.t25

arr_map
dist2<0.009
SELECT *
FROM gaia.dr3lite
JOIN tap_upload.t33
USING (source_id)

SELECT
  tc.*, otype
FROM basic AS db
 RIGHT OUTER JOIN TAP_UPLOAD.t34 AS tc
 ON 1=CONTAINS(POINT('ICRS', db.ra, db.dec),
               CIRCLE('ICRS', tc.ra, tc.dec, 1./3600.))

nflux[4:7]
a[:3]+a[3:] == a
arr_slice
WR*
SELECT otype, label
FROM otypedef
WHERE description LIKE '%Wolf%'

gaiadr3.xp_continuous_mean_spectrum
SELECT a.ssa_location, a.accref, b.accref
FROM
  gdr3spec.ssameta AS a
  JOIN lamost6.ssa_lrs AS b
  ON DISTANCE(a.ssa_location, b.ssa_location)<0.001

DISTANCE(.,.)<radius
1=CONTAINS(POINT,
CIRCLE)
ssa_location[0]
ssa_location[1]
accref
Form
accref_
SELECT
  source_id, ra, dec, parallax, phot_g_mean_mag,
  phot_bp_mean_mag, phot_rp_mean_mag, ruwe, adoptedrv,
  flux, flux_error
FROM gcns.main
JOIN gdr3spec.spectra
USING (source_id)
WHERE phot_rp_mean_mag<phot_bp_mean_mag-4

source_id
gdr3spec.withpos
flux
sequence(41, 400, 10)
add(multiply(10,sequence(41)),400)
prdust.map_union
healpix
best_fit
best_fit[20]
SELECT TOP 20 healpix
FROM prdust.map_union
ORDER BY best_fit[20] ASC

SELECT TOP 30 healpix, extinction
FROM (
  SELECT healpix, best_fit[20] as extinction
  FROM prdust.map_union) AS q
ORDER BY extinction ASC

# healpix extinction
1021402 0.00479
1021403 0.0068
418619  0.00707
...

ivo_healpix_center
ivo_healpix_center(hpxOrder INTEGER, hpxIndex BIGINT) -> POINT

  returns a POINT corresponding to the center of the healpix with the
  given index at the given order.

SELECT TOP 30 ivo_healpix_center(hpx_order, healpix) AS pos, extinction
FROM (
  SELECT healpix, best_fit[20] as extinction, hpx_order
  FROM prdust.map_union) AS q
ORDER BY extinction ASC

# pos                                    extinction
"(42.27822580645164, 78.65148926014334)" 0.00479
"(42.44939271255061, 78.6973986631694)"  0.0068
"(58.97460937500027, 40.86635677386179)" 0.00707
...

gavo_
SELECT TOP 30
  gavo_transform('GALACTIC', 'ICRS',
    ivo_healpix_center(hpx_order, healpix)) AS pos,
  extinction
FROM (
  SELECT healpix, best_fit[20] as extinction, hpx_order
  FROM prdust.map_union) AS q
ORDER BY extinction ASC

# pos                                    extinction
"(205.6104289782676, 28.392541949473785)" 0.00479
"(205.55600830161907, 28.42330388161418)" 0.0068
"(250.47595812552925, 36.43011215633786)" 0.00707
"(166.10872483007287, 21.232866316024364)" 0.00714
"(259.3314211312357, 43.09275090468469)" 0.00742
"(114.66957763676628, 21.603135736808532)" 0.00787
"(229.69174233173712, 2.0244022486718793)" 0.00793
"(214.85349325052758, 33.6802370378023)" 0.00804
"(204.8352084989552, 36.95716352922782)" 0.00806
"(215.95667870050661, 36.559656879148044)" 0.00839
"(229.66068062277128, 2.142516479012763)" 0.0084
"(219.72263539838667, 58.371829835018424)" 0.00844
...

zless
/usr/share/doc/python3-gavo/changelog.gz
ivoa.obscore
em_min
em_max
8000 .. 10000
my_col
my_col::double precision < 4

dachs
limits //obscore
displayUnit
<column name="e_min"
  unit="J"
  description="Lower energy in the spectrum"
  displayHint="spectralUnit=Angstrom"/>

root-tree.html
tmpl_resDetails
/var/gavo/web/templates/root.html
$description
$!description
resdir="."
resource
inputsDir/<schema>
\resdir
index
option
<STREAM id="staticindex">
  <doc>For indexes on tables that never change, save about 10% storage
  by feeding this.</doc>
  <option>WITH (fillfactor=100)</option>
</STREAM>

<STREAM id="onfastdisk">
  <doc>FEED this into an index to let it live on a fast disk</doc>
  <option>TABLESPACE fast</option>
</STREAM>

<index columns="foo">
  <FEED source="%#staticindex"/>
  <FEED source="%#onfastdisk"/>
</index>

/etc/gavo.rc
makeIAUId(prefix, long, lat)
utils.formatFloat
statistics
enumerate
-d
dachs limits
CAST(\em_min AS real) AS em_min,
CAST(\em_max AS real) AS em_max,

\em_min
\em_min
test
xsdvalidate
I-OBS-QSUB-5 Submitting query: SELECT TOP 1 obs_id FROM ivoa.ObsCore WHERE obs_id IS NULL
E-OBS-QERR-1 TAP query failed [Service error: "Field query: Query timed out (took too long).

obs_id
ivoa.obscore
obs_id
lower/upper
lower upper
ivoa.obscore
SELECT ...
FROM ivoa.obscore
JOIN ivoa.obs_visibility
  USING (obs_publisher_did)
WHERE (some visiblity-specific constraint)

ivoa.obscore
obs_publisher_did
obs_id
obs_id
obs_id

--- gavo/web/vodal.py (revision 8451)
+++ gavo/web/vodal.py (working copy)
@@ -260,7 +260,6 @@
        version = "1.0"
        parameterStyle = "dali"
        standardId = "ivo://ivoa.net/std/ConeSearch"
-     defaultOutputFormat = "votable1.1"

virtualenv --system-site-packages try-discoverdata
. try-discoverdata/bin/activate
cd try-discoverdata
git clone https://github.com/msdemlei/pyvo
cd pyvo
git checkout add-discoverdata
python3 setup.py develop
ipython3 notebook

stat.error;phot.mag.b
phot.mag.b;stat.error
1e-10 m
dachs val
spectralUnit
0.1nm
0.1 nm
stilts taplint
systemctl restart dachs
COALESCE(arg1, arg2, ...)
COALESCE
SELECT TOP 1 obs_publisher_did
FROM ivoa.obscore
WHERE distance(s_ra, s_dec, 83.8,-5.4)<0.2

sql_query
psql gavo
EXPLAIN SELECT obs_publisher_did
FROM ivoa.obscore
WHERE  q3c_join(83.8, - 5.4, s_ra, s_dec, 0.2) LIMIT 1;

Limit  (cost=10000000000.00..10156565675.53 rows=1 width=57) (actual time=6206.883..6206.899 rows=1 loops=1)
[...]
 Planning Time: 22.174 ms
 JIT:
   Functions: 130
   Options: Inlining true, Optimization true, Expressions true, Deforming true
   Timing: Generation 55.404 ms, Inlining 107.280 ms, Optimization 3479.626 ms, Emission 2601.411 ms, Total 6243.721 ms
 Execution Time: 6263.243 ms

jit_above_cost = 20000000000

postgresql.conf
/etc/postgresql/13/main/
listen_addresses = '*'
pg_hba.conf
max_connections = 200
localtime = UTC
shared_buffers = 15GB
/etc/sysctl.d/30-postgresql-shm.conf
temp_buffers = 100MB
work_mem = 64MB
maintenance_work_mem = 2048MB
SET enable_seqscan=false
dachs limits
dachs limits q
<column>
gavo mkrd
dachs
start
dachs adm suggestucds
metadata
superseded
<meta name="superseded" format="rst">
  We do not publish Gaia DR1 data here any more.
  If you actually need DR1 data, refer to the
  full Gaia mirrors, for instance `the one at
  ARI`_.  Otherwise, please use more recent data
  releases, for instance `eDR3`_.

  .. _the one at ARI: http://gaia.ari.uni-heidelberg.de
  .. _eDR3: /browse/gaia/q3
</meta>

python3
ivo_specconv
gavo_specconv(expr,
target_unit)
expr
target_unit
SELECT TOP 5 gavo_specconv(em_min, 'GHz') AS nu
FROM ivoa.obscore
WHERE gavo_specconv((em_min+em_max)/2, 'GHz')
    BETWEEN 1 AND 2
  AND obs_collection='VLBA LH sources'

obs_collection
SELECT TOP 5 access_url, gavo_specconv(em_min, 'keV') AS energy
FROM ivoa.obscore
WHERE gavo_specconv((em_min+em_max)/2, 'keV')
  BETWEEN 0.5 AND 2
  AND obs_collection='RASS'

gavo_specconv(656, 'Hz')
nu between 3e8/em_min and 3e7/em_max
3e8/nu between em_min and em_max
gavo_specconv
gavo_specconv(expr, unit_of_expr, target_unit)
SELECT TOP 5 gavo_specconv(em_min, 'Angstrom') AS nu
FROM ivoa.obscore
WHERE gavo_specconv(5000, 'Angstrom', 'm')
  BETWEEN em_min AND em_max

SELECT gavo_specconv(
  (spectral_start+spectral_end)/2, 'daerg')
  AS energy
FROM rr.stc_spectral
WHERE gavo_specconv(0.0002, 'apc', 'J')
  BETWEEN spectral_start AND spectral_end

def toStdUnit(fromUnit):
    for stdUnit in ["J", "Hz", "m"]:
        try:
             factor = base.computeConversionFactor(
                 fromUnit, stdUnit)
        except base.IncompatibleUnits:
            continue
        return stdUnit, factor

    raise common.UfuncError(
        f"specconv: {fromUnit} is not a spectral unit understood here")

base.computeConversionFactor
IncompatibleUnits
SPEC_CONVERSION = {
    ("J", "m"): "h*c/(({expr})*{f})",
    ("J", "Hz"): "({expr})*{f}/h",
    ("J", "J"): "({expr})*{f}",
    ("Hz", "m"): "c/({expr})/{f}",
    ("Hz", "Hz"): "{f}*({expr})",
    ("Hz", "J"): "h*{f}*({expr})",
    ("m", "m"): "{f}*({expr})",
    ("m", "Hz"): "c/({expr})/{f}",
    ("m", "J"): "h*c/({expr})/{f}",}

expr
f
expr
expr
toStdUnit
cstl.geo
p
SELECT * FROM
cstl.geo
WITH nearest AS (
SELECT TOP 100
  a.source_id,
  a.ra, a.dec,
  phot_g_mean_mag,
  dist_50,
  spectral
FROM gcns.main AS a
LEFT OUTER JOIN hdgaia.main AS b
  ON (b.source_id_dr3=a.source_id)
ORDER BY dist_50 ASC)
SELECT nearest.*, name
FROM nearest
JOIN cstl.geo AS g
  ON (1=CONTAINS(
    POINT(nearest.ra, nearest.dec),
    p))

SELECT
  COUNT(*) AS n,
  ROUND(uvel_50/5)*5 AS ubin,
  ROUND(vvel_50/5)*5 AS vbin,
  ROUND(wvel_50/5)*5 AS wbin
FROM gcns.main
GROUP BY ubin, vbin, wbin

shading=density densemap=inferno densefunc=linear
n
zw.vot
zw.sh
#!/bin/sh
stilts plot2cube \
 xpix=887 ypix=431 \
 xlabel='ubin / km/s' ylabel='vbin / km/s' \
 zlabel='wbin / km/s' \
 xmin=-184.5 xmax=49.5 ymin=-77.6 ymax=57.6 \
 zmin=-119.1 zmax=94.1 phi=-84.27 theta=90.35 \
  psi=-62.21 \
 auxmin=1 auxmax=53.6 \
 auxvisible=true auxlabel=n \
 legend=true \
 layer=Mark \
    in=zw.vot \
    x=ubin y=vbin z=wbin weight=n \
    shading=weighted size=2 color=blue

sh zw.sh
stilts tloop
import sys
from astropy import table
import numpy

angles = numpy.array(
  [float(a) for a in range(0, 360)])
table.Table([
    angles,
    40+30*numpy.cos((angles+57)*numpy.pi/180)],
  names=("psi", "theta")).write(
    sys.stdout, format="votable")

names
animate=:loop:0,360,0.5
acmd='addcol phi $1'
acmd='addcol theta 40+30*cosDeg($1+57)'

plot2cube
OUTPUT="omode=out out=pre-movie.png"
#OUTPUT=omode=swing

python3 camera.py |\
stilts plot2cube \
   xpix=500 ypix=500 \
   xlabel='ubin / km/s' ylabel='vbin / km/s' \
   zlabel='wbin / km/s' \
   xmin=-184.5 xmax=49.5 ymin=-77.6 ymax=57.6 \
   zmin=-119.1 zmax=94.1 \
   auxmin=1 auxmax=53.6 \
phi=8 \
animate=- \
afmt=votable \
$OUTPUT \
   layer=Mark \
      in=zw.vot \
      x=ubin y=vbin z=wbin weight=n \
      shading=weighted size=4 color=blue

# render to movie with something like
# ffmpeg -i "pre-movie-%03d.png" -framerate 15 -pix_fmt yuv420p /stream-movie.webm
# (the yuv420p incantation is so real-world
# web browsers properly will not go psychedelic
# with the colours)

def get_roots_for(term, uat_terms):
  roots, seen = set(), set()

  def follow(t):
    wider = uat_terms[t]["wider"]
    if not wider:
      if not t in ROOT_TERMS:
        raise Exception(
          f"{t} found as a top-level term")
      roots.add(t)
    else:
      seen.add(t)
      for wider in uat_terms[t]["wider"]:
        follow(wider)

  follow(term)
  return roots

uat_terms
def addUATToplevels(dataciteTree):
  # dataciteTree is an (lxml) ElementTree for the
  # result of the XSL transformation.  That's all
  # I have, and thus I first have to fiddle out
  # the identifier we are talking about
  ivoid =  dataciteTree.xpath(
      "//d:alternateIdentifier["
      "@alternateIdentifierType='ivoid']",
      namespaces={"d": DATACITE_NS}
    )[0].text.lower()
  # The .lower() is necessary because ivoids
  # unfortunately are case-insensitive, and RegTAP
  # normalises them to lowercase to retain sanity.

  # Now pull the UAT-mapped subject keywords from
  # our RegTAP extension (getTableConn is
  # DaCHS-internal API, but there's no magic in
  # there, it's just connection pooling with
  # guarantees against connections  idle in
  # transaction).
  with base.getTableConn() as conn:
    subjects = set(r[0] for r in
      conn.query("SELECT uat_concept"
        " FROM rr.subject_uat"
        " WHERE ivoid=%(ivoid)s", locals()))

  # This is the mapping itself: we do
  # roots-subjects to avoid adding
  # root terms that are already in
  # the record itself.  UAT_TOPLEVELS is the result
  # of the root finding discussed above.
  for term in subjects:
    root = UAT_TOPLEVELS[term]
    newRoots |= (root-subjects)

  # And finally fiddle in any new root terms found
  # into the datacite tree
  if newRoots:
    subjects = dataciteTree.xpath(
      "//d:subjects",
      namespaces={"d": DATACITE_NS})[0]
    for root in newRoots:
      newSubject = etree.SubElement(subjects,
        f"{{{DATACITE_NS}}}subject")
      newSubject.text = root

SELECT TOP 500
  ra, dec, source_id, phot_g_mean_mag, ruwe,
  r_med_photogeo,
  partner_id, dist,
  COORD2(gavo_transform('ICRS', 'GALACTIC',
    point(ra, dec))) AS glat
FROM
  gedr3dist.litewithdist
  NATURAL JOIN gedr3auto.main
ORDER BY dist DESC

NATURAL JOIN
JOIN USING
gavo_transfrom
gedr3dist.litewithdist
gaiaedr3.gaiasource
gaia.edr3lite
SELECT TOP 300
  a.source_id, partner_id, dist,
  a.phot_g_mean_mag AS source_mag,
  b.phot_g_mean_mag AS partner_mag,
  abs(a.phot_g_mean_mag-b.phot_g_mean_mag) AS magdiff
FROM gedr3auto.main
  NATURAL JOIN gaia.edr3lite AS a
  JOIN gaia.edr3lite AS b
    ON (partner_id=b.source_id)
WHERE
  dist BETWEEN 9.999/3600 AND 10.001/3600
  AND a.phot_g_mean_mag IS NOT NULL
  AND b.phot_g_mean_mag IS NOT NULL
ORDER BY magdiff ASC

dc.users
.unload(
.on("unload"
completeURL("/logo_tiny.png")
sources
<sources
pattern="data/202?/*.fits"/>
/var/gavo/web/nv_static/img
setup/@imports
dc.metastore
table.query()
querier.query()
table.query
table.expand(query)
gavo=$ select r_med_geo from gedr3dist.main
gavo-$ where source_id=563018673253120;
    1430.9

gavo=$ set extra_float_digits=3;
gavo=$ select r_med_geo from gedr3dist.main
gavo-$ where source_id=563018673253120;
 1430.90332

dachs --version
alter role (you) set extra_float_digits=3,

bysemantics
gavo_vocmatch
SELECT TOP 5 *
FROM ivoa.obscore
WHERE
  1=gavo_vocmatch(
    ’product-type’,
    ’timeseries’,
    dataproduct_type)

waveband
gavo_vocmatch
dachs datapack
dachs datapack create myres/q myres.pack
dachs datapack load myres.pack
dachs val -vc ALL
ALL_RECURSE
ALL
[web]root
<schema
foo="bar">my_schema<schema>
style
Points
Lines
usingIndex
SELECT name, deepest_shape
FROM openngc.shapes

SELECT
  obj_type,
  shape,
  AREA(shape) AS ar
FROM (
  SELECT obj_type, SUM(deepest_shape) AS shape
  FROM openngc.shapes
  NATURAL JOIN openngc.data
  GROUP BY obj_type) AS q

area
sum
SELECT ivoid, res_title, coverage
FROM rr.subject_uat
  NATURAL JOIN rr.stc_spatial
  NATURAL JOIN rr.resource
WHERE uat_concept='planetary-nebulae'
  AND AREA(coverage)<20

SELECT
  ivoid,
  res_title,
  gavo_mocintersect(coverage, dfbscoverage) as ovrlp
FROM (
  SELECT ivoid, res_title, coverage
  FROM rr.subject_uat
  NATURAL JOIN rr.stc_spatial
  NATURAL JOIN rr.resource
  WHERE uat_concept='planetary-nebulae'
  AND AREA(coverage)<20) AS others
CROSS JOIN (
  SELECT coverage AS dfbscoverage
  FROM rr.stc_spatial
  WHERE ivoid='ivo://org.gavo.dc/dfbsspec/q/spectra') AS dfbs

gavo_mocintersect
gavo_mocunion
sum
SELECT
  im.*,
  gavo_mocintersect(MOC(6, im.coverage), pn_coverage) as ovrlp
FROM (
  SELECT SUM(coverage) AS pn_coverage
  FROM rr.subject_uat
  NATURAL JOIN rr.stc_spatial
  WHERE uat_concept='planetary-nebulae'
  AND AREA(coverage)<20) AS c
JOIN lsw.plates AS im
ON 1=INTERSECTS(pn_coverage, MOC(6, coverage))

MOC(order, geo)
SELECT TOP 1
  MOC('4/30-33 38 52 7/324-934') AS ar
FROM tap_schema.tables

cmoc
coverage
SELECT db.* FROM
  lsw.plates AS db
  JOIN tap_upload.t6
ON 1=CONTAINS(coverage, cmoc) -- fails!

SELECT db.* FROM
  lsw.plates AS db
  JOIN tap_upload.t6
ON 1=CONTAINS(MOC(7, coverage), cmoc)

sudo apt install postgresql-pgsphere
dachs upgrade -e
dachs upgrade -e | psql gavo   # as dachsroot

select top 1 gavo_mocunion(moc('1/3'), moc('2/9'))
from tap_schema.tables

gavo_mocintersect(moc('1/3'), moc('2/13'))
ivo_healpix_index
GROUP BY
...ROUND(r_est/20) as bin GROUP by name,
bin
gavo_histogram(val REAL, lower REAL, upper REAL, nbins INTEGER) -> INTEGER[]

The aggregate function returns a histogram of val with
nbins+2 elements. Assuming 0-based arrays, result[0] contains
the number of underflows (i.e., val<lower), result[nbins+1]
the number of overflows. Elements 1..nbins are the counts in
nbins bins of width (upper-lower)/nbins. Clients will have to
convert back to physical units using some external communication,
there currently is no (meta-) data as to what lower and upper was in
the TAP response.

gavo_histogram(r_est, 0, 2000, 20) as hist
select name, raj2000, dej2000, maj_ax_deg
from openngc.data
where obj_type='OCl'

select name, source_id
from openngc.data
join gaia.dr2light
on (
  1=contains(
    point(ra,dec),
    circle(raj2000, dej2000, maj_ax_deg)))
where obj_type='OCl')

source_id
r_est
with ocl as (
  select name, raj2000, dej2000, maj_ax_deg, source_id
  from openngc.data
  join gaia.dr2light
  on (
    1=contains(
      point(ra,dec),
      circle(raj2000, dej2000, maj_ax_deg)))
  where obj_type='OCl')

select
  name,
  gavo_histogram(r_est, 0, 4000, 200) as hist
from
  gdr2dist.main
  join ocl
  using (source_id)
where r_est!='NaN'
group by name

r_est!='NaN'
gavo_histogram
val!=val
ocl.vot
GALLERY = True
from astropy.table import Table
import matplotlib.pyplot as plt
import numpy
from scipy.interpolate import UnivariateSpline

GALLERY = False

def substract_background(arr):
    x = range(len(arr))
    mean = sum(arr)/len(arr)
    arr = arr/mean
    background = UnivariateSpline(x, arr, s=100)
    cleaned = arr-background(x)

    if GALLERY:
        plt.plot(x, arr)
        plt.plot(x, background(x))
        plt.show()

    return cleaned


def main():
    tab = Table.read("ocl.vot")
    hist = numpy.array([substract_background(r["hist"][1:-1])
      for r in tab])
    plt.matshow(hist, cmap='gist_heat')
    plt.show()


if __name__=="__main__":
    main()

git diff
--shortstat
apt install gavodachs2-server
dachs info
--sample-percent
dachs imp -I q
dachs limits
[web]adaptProtocol
[ivoa]validOAISets
n:data="some_key"
n:data="key some_key" n:render="str"
/var/gavo/web/templates/root.html
<ul
n:data="chunk"
<ul
n:data="key chunk"
SELECT ROUND(amp/err_mag*10)/10 AS bin, COUNT(*) AS n
FROM bgds.phot_all
WHERE nobs>10
GROUP BY bin

WITH photpoints AS (
  SELECT
    amp/err_mag AS redamp,
    amp,
    ivo_healpix_index(5, ra, dec) AS hpx
  FROM bgds.phot_all
  WHERE
    nobs>10
    AND band_name='SDSS i'
    AND mean_mag<16),
all_objs AS (
  SELECT count(*) AS ct,
    hpx
    FROM photpoints GROUP BY hpx),
strong_var AS (
  SELECT COUNT(*) AS ct,
    hpx
    FROM photpoints
    WHERE redamp>4 AND amp>1 GROUP BY hpx)
SELECT
  strong_var.ct/(1.0*all_objs.ct) AS obs,
  all_objs.ct AS n,
  hpx
FROM strong_var JOIN all_objs USING (hpx)
WHERE all_objs.ct>20

SELECT ra, dec, main_id
FROM basic
WHERE
  otype='LP?'
  AND 1=CONTAINS(
     POINT('', ra, dec),
     POLYGON('', 127, -30, 112, -30, 272, -30, 258, -30))

max_parallel_workers_per_gather = 4
max_worker_processes = 10
max_parallel_workers = 10

max_parallel_workers_per_gather
max_worker_processes
import contextlib
import pyvo
import time

@contextlib.contextmanager
def timeit(activity):
  start_time = time.time()
  yield
  end_time = time.time()
  print("Time spent on {}: {} s".format(activity, end_time-start_time))


svc = pyvo.tap.TAPService("http://dc.g-vo.org/tap")
with timeit("Cold (?) run"):
  svc.run_sync("select round(Rmag) as bin, count(*) as n"
    " from ppmx.data group by bin")
with timeit("Warm run"):
  svc.run_sync("select round(Rmag) as bin, count(*) as n"
    " from ppmx.data group by bin")

SELECT
        count(*) AS obs,
        source_id/140737488355328 AS hpx
FROM gaia.dr2light
GROUP BY hpx

max_parallel_workers_per_gather
set max_parallel_workers_per_gather=0;

explain analyze select round(Rmag) as bin, count(*) as n from ppmx.data group by bin;

max_parallel_workers_per_gatherer
max_worker_processes
/etc/postgresql/13/main/postgresql.conf
parallel_tuple_cost = 0.001
parallel_setup_cost = 3

select
  ssa_pubDID, accref, raj2000, dej2000, ssa_targsubclass
from lamost5.data tablesample(1)
where
  ssa_targsubclass like 'M%'

ssa_pubdid
iter_datalinks
import pyvo

svc = pyvo.dal.TAPService("http://dc.g-vo.org/tap")
for dl in svc.run_sync("select top 3 ssa_pubdid"
        " from lamost5.data").iter_datalinks():
    print(next(dl.bysemantics("#progenitor")
        )["access_url"].decode("ascii"))

from astropy import units as u
proc = next(dl.iter_procs())
cutout = proc.processed(band=(392*u.nm,398*u.nm))

def spectral_model(wl, c1, c2, depth, width):
    return (1
        -depth*numpy.exp(-numpy.square(wl-c1)
            /numpy.square(width))
        -depth*numpy.exp(-numpy.square(wl-c2)
            /numpy.square(width)))

for pubdid, prof in get_profiles(
        "G8", (392*u.nm,398*u.nm), 0.01, 4):
    prof["flux"] /= max(prof["flux"])
    popt, pcov = curve_fit(
        spectral_model, prof["spectral"], prof["flux"],
        sigma=prof["flux_error"],
        p0=[3968, 3934, 1, 1])
    if pcov[3][3]>1:
        break

get_profiles
b"a"!="a"
deb http://vo.ari.uni-heidelberg.de/debian buster-foreports main

apt install postgresql-q3c
postgresql-11-pgsphere
reindex database gavo

/etc/postgresql/11/main/pg_hba.conf
host    all         all         ::1/32          md5

select * from rr.interface
where
  intf_type='vr:paramhttp'
  and not intf_role='std'

n = float('nan');print(n==n)
select * from tap_schema.schemas where NULL=NULL

select * from tap_schema.schemas where NULL!=NULL

select count(*) from openngc.data where mag_b<10

select count(*) from openngc.data where not mag_b<10

select count(*) from openngc.data where mag_b>=10

select count(*) from openngc.data where mag_b is null

select count(*) from openngc.data
where
  not mag_b<10
  or mag_b is null

IS NULL
IS
NOT NULL
hidden="True"
[web]maxSyncUploadSize=500000
maxSyncUploadSize=20000000
source_table
preview
arraysize="1"
gavo_transform
gavo_ipix
gavo_urlescape
gavo_urlescape
CatalogResource
//soda#sdm_genDesc
useAccref="False"
descriptorGenerator
--foreground
Column t_exptime: 3460 values
  Min   12, Max 15300, Mean 890.24  [previous mean: 370.722]
---
Column em_mean: 3801 values
  Min 1.8081e-09, Max 9.3e-07, Mean 6.40804e-07 [No change: Sigh!]
---
Column t_mean: 4731 values
  Min 12564.5, Max 58126.3, Mean 49897.9 [previous mean: 51909.1]
---
Column instrument_name: 4747 values
  Matches from , Petzval, [Max Wolf's residence in
  Heidelberg, Maerzgasse, Wolf's Doppelastrograph,
  Heidelberg Koenigstuhl (24), Wolf's
  Doppelastrograph,] AG-Astrograph, [Zeiss Triplet
  15 cm Potsdam-Telegrafenberg], Zeiss Triplet,
  Astrograph (four 10-cm Tessar f/6 cameras),
  [3.5m APO, ROSAT PSPCC, Heidelberg Koenigstuhl
  (24), Bruce Astrograph, Calar Alto (493),
  Schmidt], Grosser Refraktor, [ROSAT HRI,
  DK-1.54], Hamburger Schmidt-Spiegel,
  [DFOSC_FASU], ESO 1-metre Schmidt telescope,
  Great Schmidt Camera, Lippert-Astrograph, Ross-B
  3", [AZT 22], Astrograph (six 10-cm Tessar f/6
  cameras), 1m-Spiegelteleskop, [ROSAT PSPCB],
  Astrograph (ten 10-cm Tessar f/6 cameras), Zeiss
  Objective
---
Column access_url: 4747 values [4067]

SELECT t_min, access_url, s_region
FROM ivoa.obscore
WHERE instrument_name like '%Tessar%'
AND 1=CONTAINS(POINT(345, -38), s_region)

select spectral,
  avg(phot_bp_mean_mag-phot_rp_mean_mag) as col,
  count(*) as ct
from hdgaia.main
join gaia.dr2light
using (source_id)
group by spectral

select spectral,
  avg(phot_bp_mean_mag-phot_rp_mean_mag) as col,
  sqrt(avg(power(phot_bp_mean_mag-phot_rp_mean_mag, 2))-
    power(avg(phot_bp_mean_mag-phot_rp_mean_mag), 2)) as sig_col,
  count(*) as ct
from hdgaia.main
join gaia.dr2light
  using (source_id)
where m_v<18
group by spectral
having count(*)>10

with standards as (select spectral,
  avg(phot_bp_mean_mag-phot_rp_mean_mag) as col,
  sqrt(avg(power(phot_bp_mean_mag-phot_rp_mean_mag, 2))-
    power(avg(phot_bp_mean_mag-phot_rp_mean_mag), 2)) as sig_col,
  count(*) as ct
  from hdgaia.main
  join gaia.dr2light
  using (source_id)
  where m_v<18
  group by spectral
  having count(*)>10)
select *
from hdgaia.main
join standards
using (spectral)
join gaia.dr2light using (source_id)
where
  abs(phot_bp_mean_mag-phot_rp_mean_mag-col)>3*sig_col
  and m_v<18

[web]
serverURL: http://dc.zah.uni-heidelberg.de
alternateHostnames:dc.g-vo.org, vo.uni-hd.de

/etc/gavo.rc
prdust.map_union
coverage
best_fit
grdiagnostic
coverage
best_fit
grdiagnostic
best_fit[ind]
best_fit[ind]
grdiagnostic[ind]
gaiadr2_complements.geometric_distance
source_id
SELECT
r_est, 5*log10(r_est)-5 as dist_mod,
phot_g_mean_mag, phot_bp_mean_mag, phot_rp_mean_mag,
ra, dec
FROM
gaiadr2.gaia_source
JOIN gaiadr2_complements.geometric_distance
USING (source_id)
WHERE
parallax_over_error>1
AND 1=CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', 288.5, 2.3, 0.5 ))

phot_bp_mean_mag-phot_rp_mean_mag
phot_g_mean_mag-dist_mod
prdust.map_union
ivo_healpix_index
gavo_transform
map_union
CAST(ivo_healpix_index(10,
  gavo_transform('ICRS', 'GALACTIC', POINT(ra, dec))) AS INTEGER)

ivo_healpix_index
ivo_interval_overlaps
WITH sources AS (
  SELECT phot_g_mean_mag,
    phot_bp_mean_mag,
    phot_rp_mean_mag,
    dist_mod,
    CAST(ivo_healpix_index(10,
      gavo_transform('ICRS', 'GALACTIC', POINT(ra, dec))) AS INTEGER) AS hpx,
    ROUND((dist_mod-4)*2)+1 AS dist_mod_bin
  FROM TAP_UPLOAD.T1)

SELECT
  phot_bp_mean_mag-phot_rp_mean_mag-dust.best_fit[dist_mod_bin] AS color,
  phot_g_mean_mag-dist_mod+
    dust.best_fit[dist_mod_bin]*3.384 AS abs_mag,
  dust.grdiagnostic[dist_mod_bin] as qual
FROM sources
JOIN prdust.map_union AS dust
ON (1=ivo_interval_has(hpx, coverage))

dachs val ALL; apt update; apt upgrade
release
beta
deb http://vo.ari.uni-heidelberg.de/debian release main

dachs start
dachs imp //adql
gavo_histogram
coverage
sources
_associatedDatalinkSvc
datalink
/var/gavo/etc/matplotlibrc
run dachs
init
dachs imp
-I
<property
name="previews">auto</property>
statisticsTarget
<property key="
statisticsTarget">10000</property>
gavo imp -m
[web]logFormat
/etc/gavo.rc
dachs imp -m //obscore
SELECT *
gaia.dr2light
gaia_source
gaia_source
gaia.dr2epochflux
gaia_source
SELECT g.*, g_transit_time, g_transit_flux
FROM gaia.dr2light AS g
LEFT OUTER JOIN gaia.dr2epochflux
USING (source_id)
WHERE ...whatever...

LEFT OUTER JOIN
g_transit_time
g_transit_flux
from matplotlib import pyplot as plt
import pyvo

def main():
  simbad = pyvo.dal.TAPService(
    "http://simbad.u-strasbg.fr:80/simbad/sim-tap")
  gavodc = pyvo.dal.TAPService("http://dc.g-vo.org/tap")

  # Get blue supergiants from Simbad
  simjob = simbad.submit_job("""
    select main_id, ra, dec
    from basic
    where otype='BlueSG*'""")
  simjob.run()

  # Get lightcurves from Gaia
  try:
    simjob.wait()
    time_series = gavodc.run_sync("""
      SELECT b.*, bp_obs_time, bp_flux, rp_obs_time, rp_flux
      FROM (SELECT
         main_id, source_id, g.ra, g.dec
         FROM
        gaia.dr2light as g
         JOIN TAP_UPLOAD.t1 AS tc
         ON (0.002>DISTANCE(tc.ra, tc.dec, g.ra, g.dec))
      OFFSET 0) AS b
      JOIN gaia.dr2epochflux
      USING (source_id)
      """,
      uploads={"t1": simjob.result_uri})
  finally:
    simjob.delete()

  # Now plot one after the other
  for row in time_series.table:
    plt.plot(row["bp_obs_time"], row["bp_flux"])
    plt.plot(row["rp_obs_time"], row["rp_flux"])
    plt.show(block=False)
    raw_input("{}; press return for next...".format(row["main_id"]))
    plt.cla()

if __name__=="__main__":
  main()

gaia.dr2epochflux
  SELECT source_id, ra, dec,
  phot_bp_mean_mag, phot_rp_mean_mag, phot_g_mean_mag,
  g_transit_time, g_transit_flux,
  rp_obs_time, rp_flux
  FROM gaia.dr2epochflux
  JOIN gaia.dr2light
  USING (source_id)
  WHERE parallax>50

into “ADQL” text to retrieve lightcurves for the more nearby
variables (in reality, you'd have to be a bit more careful with the
distances, but you already knew that).

phot_bp_mean_mag-phot_rp_mean_mag
phot_g_mean_mag
SELECT TOP 100
  source_id, ra, dec, parallax, g.pmra, g.pmdec,
  m.name, m.pmra AS c_pmra, m.pmde AS c_pmde,
  m.e_pm AS c_e_pm,
  1/dist AS cluster_parallax
FROM
  gaia.dr2epochflux
  JOIN gaia.dr2light AS g USING (source_id)
  JOIN mwsc.main AS m
  ON (1=CONTAINS(
    POINT(g.ra, g.dec),
    CIRCLE(m.raj2000, m.dej2000, rcluster)))
WHERE IN_UNIT(pmdec, 'deg/yr') BETWEEN m.pmde-m.e_pm*3 AND m.pmde+m.e_pm*3

dachs start
$ dachs start list
siap -- Image collections via SIAP1 and TAP
ssap+datalink -- Spectra via SSAP and TAP, going through datalink
epntap -- Solar system data via EPN-TAP 2.0
scs -- Catalogs via SCS and TAP

dachs start
$ mkdir ex_data
$ cd ex_data
$ dachs start scs
$ head -16 q.rd | tail -9
<resource schema="ex_data">
  <meta name="creationDate">2018-04-13T12:34:31Z</meta>

  <meta name="title">%title -- not more than a line%</meta>
  <meta name="description">
    %this should be a paragraph or two (take care to mention salient terms)%
  </meta>
  <!-- Take keywords from
    http://astrothesaurus.org/thesaurus/hierarchical-browse/

dachs start
q.rd
augroup rd
  au!
  au BufRead,BufNewFile *.rd imap  /%[^%]*%a
  au BufRead,BufNewFile *.rd imap  cf%
augroup END

~/.vimrc
DISTANCE(ra1, dec1, ra2, dec2)<dist
1=CONTAINS(POINT(), CIRCLE())
POINT('ICRS', ra, dec)
CONTAINS(point_in_icrs, circle_in_galactic)
POINT('GALACTIC', ra, dec)
POINT(long, lat)
CAST
CAST('1991-02-01', TIMESTAMP)
TIMESTAMP(tx)
CAST(tx, TIMESTAMP)
IN_UNIT
IN_UNIT(pmra, 'deg/yr')
     SELECT <whatever complex> FROM
         (SELECT a,b,c FROM source
           WHERE <crazy stuff>
           GROUP BY a, b, c) as left
       UNION
         (SELECT a,b,c FROM source
           WHERE <other crazy stuff>
           GROUP BY a, b, c) as right
     WHERE <more complex stuff over a, b, and c>

– and similarly, EXCEPT lets you “punch a hole” in a result table.
Another interesting use case would be to query many tables on a
service like VizieR in one go; that still works if you make sure the
tables defined by the sub-queries have the same columns. Given that a
lot of cross-table operations actually boil down to JOINs and WHERE
clauses, the set operations are used less that one would expect. But
if you need them, there's no real alternative (short of downloading
far too much and performing the operation locally, which of course
defeats the purpose of TAP).

MOD(flags,16)>7
BITWISE_AND(flags,8)=8
flags&8=8
SELECT TOP 10 * FROM hipparcos.main ORDER BY parallax DESC
SELECT TOP 10 *
FROM hipparcos.main
ORDER BY parallax DESC
OFFSET 10

OFFSET 0
CIRCLE(POINT(ra, dec), radius)
CIRCLE(ra, dec, radius)
SELECT *
FROM zcosmos.data
WHERE 0=CONTAINS(
  ssa_targetpos,
  CIRCLE(ssa_location, ssa_aperture))

WHERE is_broken=True
if is_broken==True:
if is_broken:
select ivoid from
rr.table_column
natural join rr.stc_spatial
where
  1=contains(gavo_simbadpoint('HDF'), coverage)
  and ucd like 'phot.flux;em.radio%'

SELECT ivoid FROM
rr.stc_temporal
NATURAL JOIN rr.stc_spectral
NATURAL JOIN rr.stc_spatial
WHERE
  1=CONTAINS(gavo_simbadpoint('M1'), coverage)
  AND 1=ivo_interval_overlaps(
    6.69e-7, 6.75e-7,
    wavelength_start, wavelength_end)
  AND 1=ivo_interval_overlaps(
    36900, 38800,
    time_start, time_end)

ivo_interval_overlaps
SELECT COUNT(*) FROM (
  SELECT DISTINCT ivoid FROM rr.stc_temporal) AS q

coverage
<coverage>
  <spatial>3/336,338,450-451,651-652,659,662-663
    4/1816,1818-1819,1822-1823,1829,1840-1841</spatial>
  <temporal>37190 37250</temporal>
  <temporal>54776 54802</temporal>
  <spectral>3.3e-07 6.6e-07</spectral>
  <spectral>2.0e-05 3.5e-06</spectral>
  <waveband>Optical</waveband>
  <waveband>Infrared</waveband>
</coverage>

waveband
coverage
  <updater spaceTable="data" spectralTable="data" mocOrder="4"/>
  <spectral>3.3e-07 6.6e-07</spectral>
  <temporal>37190 37250</temporal>
  <temporal>38776 38802</temporal>
  <temporal>41022 41107</temporal>
  <temporal>41387 41409</temporal>
  <temporal>41936 41979</temporal>
  <temporal>43416 43454</temporal>
  <spatial>3/282,410 4/40,323,326,329,332,387,390,396,648-650,1083,1085,1087,1101-1103,1123,1125,1132-1134,1136,1138-1139,1144,1146-1147,1173-1175,1216-1217,1220,1223,1229,1231,1235-1236,1238,1240,1597,1599,1614,1634,1636,1728,1730,1737,1739-1740,1765-1766,1784,1786,2803,2807,2809,2812</spatial>
</coverage>

select
dateobs from plts.data
updater
sourceTable
mocOrder
dachs limits q.rd
<coverage>
  <updater sourceTable="my_table"/>
</coverage>

my_table
dachs limits q.rd
pos.parallax
pos.parallax;obs
pos.parallax;stat.fit
phys.temperature
meta.code.multip
SELECT DISTINCT ivoid, table_index
FROM
rr.table_column AS t1
JOIN rr.table_column AS t2
USING (ivoid, table_index)
WHERE t1.ucd='phys.temperature'
AND t2.ucd='meta.code.multip'

rr.table_column
table_description
table_name
rr.res_table
SELECT table_name, table_description
FROM rr.res_table
NATURAL JOIN (
  SELECT DISTINCT ivoid, table_index
  FROM
  rr.table_column AS t1
  JOIN rr.table_column AS t2
  USING (ivoid, table_index)
  WHERE t1.ucd='phys.temperature'
  AND t2.ucd='meta.code.multip') as q

SELECT table_name, table_description, res_description, ivoid
FROM rr.res_table
NATURAL JOIN rr.resource
NATURAL JOIN (
  SELECT DISTINCT ivoid, table_index
  FROM
  rr.table_column AS t1
  JOIN rr.table_column AS t2
  USING (ivoid, table_index)
  WHERE t1.ucd='phys.temperature'
  AND t2.ucd='meta.code.multip') as q

dachs val ALL
dachs
upgrade
map/@source
dachs imp -m <rdid>
dachs imp //obscore
apt-get install python-gavoutils python-gavostc
SELECT * FROM
ivoa.obscore AS o
JOIN (
  SELECT top 10 * FROM antares.data
  ORDER BY n_hits desc
) AS n
ON 1=INTERSECTS(
  s_region,
  origin_est)

SELECT * FROM
ivoa.obscore AS o
JOIN antares.data as n
ON
   epoch_mjd between t_min-0.01 and t_max+0.01
  AND
    dataproduct_type='image'
  AND
    1=INTERSECTS(origin_est, s_region)

apt-get dist-upgrade
dachs
dachs imp q
gavo
man dachs
//epntap#table-2_0
//epntap2#table-2_0
imp -u
updateMode
uploadCore
updating="True"
inputTable
inputDD
inputKeys
rsc.makeData
rsc.TableForDef
with base.getTableConn() as conn:

Instrument Name	mean size [sqdeg]
Eastman Aero-Ektar K-24 Lens on a K-1...
Cerro Tololo 4 meter
Logbook Only. Pages without plates.
Roe 6-inch
Palomar Sky Survey (POSS)
1.5 inch Ross (short focus)	4284.199799877725
Patrol cameras	4220.802442888225
1.5-inch Ross-Xpress	4198.678060743206
2.8-inch Kodak Aero-Ektar	3520.3257323233624
KE Camera with Installed Rough Focus	3387.5206396388453
Eastman Aero-Ektar K-24 Lens on a K-1...	3370.5283986677637
Eastman Aero-Ektar K-24 Lens on a K-1...	3365.539790633015
3 inch Perkin-Zeiss Lens	1966.1600884072298
3 inch Ross-Tessar Lens	1529.7113188540836
2.6-inch Zeiss-Tessar	1516.7996790591587
Air Force Camera	1420.6928219265849
K-19 Air Force Camera	1414.074101143854
1.5 in Cooke "Long Focus"	1220.3028263587332
1 in Cook Lens #832 Series renamed fr...	1215.1434235932702
1-inch	1209.8102811770807
1.5-inch Cooke Lenses	1209.7721123964636
2.5 inch Cooke Lens	1160.1641223648048
2.5-inch Ross Portrait Lens	1137.0908812243645
Damons South Yellow	1106.5016573891376
Damons South Red	1103.327982978934
Damons North Red	1101.8455616455205
Damons North Blue	1093.8380971825375
Damons North Yellow	1092.9407550755682
New Cooke Lens	1087.918570304363
Damons South Blue	1081.7800084709982
2.5 inch Voigtlander (Little Bache or...	548.7147592220762
NULL	534.9269386355818
3-inch Ross Fecker	529.9219051692568
3-inch Ross	506.6278856912204
3-inch Elmer Ross	503.7932693652602
4-inch Ross Lundin	310.7279860552893
4-inch Cooke (1-327)	132.690621660727
4-inch Cooke Lens	129.39637516917298
8-inch Bache Doublet	113.96821604869973
10-inch Metcalf Triplet	99.24964308212328
4-inch Voightlander Lens	98.07368690379751
8-inch Draper Doublet	94.57937153909593
8-inch Ross Lundin	94.5685388440282
8-inch Brashear Lens	37.40061588712761
16-inch Metcalf Doublet (Refigured af...	33.61565584978583
24-33 in Jewett Schmidt	32.95324914757339
Asiago Observatory 92/67 cm Schmidt	32.71623733985344
12-inch Metcalf Doublet	31.35112644688316
24-inch Bruce Doublet	22.10390937657793
7.5-inch Cooke/Clark Refractor at Mar...	14.625992810622787
Positives	12.600189007151709
YSO Double Astrograph	10.770798601877804
32-36 inch BakerSchmidt 10 1/2 inch r...	10.675406541122827
13-inch Boyden Refractor	6.409447066606171
11-inch Draper Refractor	5.134521254785461
24-inch Clark Reflector	3.191361603405415
Lowel 40 inch reflector	1.213284257086087
200 inch Hale Telescope	0.18792105301170514

+:	I've thought long and hard about this, and I think I understand the matter in depth. You'll hence need arguments of the profundity of the Earth's outer core to sway me.
(unqualified):	I've thought about this, and as far as I understand the matter I'm sure about it. More information, solid arguments, or a sudden inspiration while showering might still sway me.
-:	This is a gut feeling. It could very well be phantom pain. Feel free to try a differential diagnosis.

HEALPix level	Integer-divide source id by
12	34359738368
11	137438953472
10	549755813888
9	2199023255552
8	8796093022208
7	35184372088832
6	140737488355328
5	562949953421312
4	2251799813685248
3	9007199254740992
2	36028797018963968

GAVO Blog: Virtual Observatory Matters

At the 2026 Strasbourg Interop

Monday 15:00 – The Local Host Session

Monday 17:00 – Apps I

Tuesday 10:00 – Science Platforms

Tuesday, 12:00 – DAL I

Tuesday 17:00 – Afternoon Sessions

Wednesday 10:00 – AI Plenary

Thursday Afternoon – After the Registry Morning

Thursday 17:20 – Spectra

Thursday Evening – In the Planetarium

Friday 12:00 – Wrapping Up

Limits, Materialisation, and Anchor Texts: DaCHS 2.13 is out

Porting a DaCHS SIAv1 service to SIAP2

Going From SIAv1 to SIAv2 in 11 Moderately Difficult Steps

Queries Against My Obscore Are Slow!

Match Your Types Before You UNION ALL

Known Problem Is Not Solved Problem

Changing Types En Masse

Success?

Out But Not Down

ADASS and Interop in Görlitz

Update: Soapbox (2025-11-12)

Update: Looking Back at the Interop (2025-11-16)

GAVO at the AG-Tagung in Görlitz

DaCHS 2.12 Is Out

At the College Park Interop

Uneasy Logistics

TCG: Come again?

Opening session (2025-06-02, 14:30)

Charge to the Working Groups (2025-06-02, 15:30)

Data Management Challenges (2025-06-03, 10:30)

Registry (2025-06-03, 12:30)

Tuesday (2025-06-03) Afternoon

DCP (2025-06-04, 10:00)

Obscore and Extensions (2025-06-04, 15:00)

Apps II (2025-06-05, 12:00)

DM 3 (2025-06-05, 17:00)

Wrapping Up (2025-06-06)

Persistent TAP Uploads Update: A Management Interface

Setting Lifetimes

Creating Indexes

Special Index Types

At the Gaia Passivation Event

9:00

9:35

9:42

9:45

9:50

9:53

9:55

12:00

12:30

13:00

A New Constraint Class in PyVO's Registry API: UAT

Thesauri and the UAT

Why Keywords?

The UAT constraint

Implementation

Doing Large-Scale ADQL Queries

Raising the Match Limit

Giving the Query More Time

Planning for Large Result Sets? Get in Contact!

Uniform-Length Partitions

Equal-Size Partitions

DaCHS 2.11: Persistent TAP Uploads

Persistent TAP Uploads

On-loaded Execute-s

dachs start taptable

VOTable 1.5

Minor Changes

Upgrade As Convenient

At the Malta Interop

The TCG discusses (Thursday, 15:00)

At the Exec Session (Thurday, 16:45)

Opening Plenary (Friday 9:30)

Self-Agency (Friday, 10:10)

Focus Session: High Energy and Time Domain (Friday, 12:00)

In Apps I (Friday 16:30)

Data Access Layer (Saturday 9:30)