2019-12-16

Power Query for Twitter API

I am lazy, so let's make it quick. Looping twitter could be a bit trouble so I put power query script (for searching a specific account by timeline API) below. Hope it helps

1. A function to get token

let
    fnGetToken = () =>
        let 
            key = "Basic " & Binary.ToText(Text.ToBinary("{you should know what it is}"),0),
            url = "https://api.twitter.com/oauth2/token",
            getJson = Web.Contents(url,
                [
                    Headers = [#"Authorization"=key,#"Content-Type"="application/x-www-form-urlencoded;charset=UTF-8"],
                    Content = Text.ToBinary("grant_type=client_credentials") 
                ]
            ),
            j = Json.Document(getJson),
            token = j[access_token],
            tokenHeader = "bearer " & token
        in tokenHeader
in
    fnGetToken


2. Another function to get the max twitter id regarding the topic

let
 fnGetMaxId = (token) =>
 let 
  GetJsonQuery = Web.Contents("https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={this is the account}&count=1",
   [
    Headers = [#"Authorization"=token]
   ]
  ),
  FormatAsJsonQuery = Json.Document(GetJsonQuery),
  t = Table.FromList(FormatAsJsonQuery, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
  maxId = Table.ExpandRecordColumn(t, "Column1", {"id_str"}, {"maxid"})
 in 
  maxId
in
 fnGetMaxId


3. the 3rd function to get result by token and endpoint. (technically you don't need it, just to make the main function easy to read)

let
 fnGetList = (token, endpoint) =>
  let 
   req = Web.Contents(endpoint,
    [
     Headers = [#"Authorization"=token]
    ]
   ),
   json = Json.Document(req)
  in 
   json
in
 fnGetList


4. finally the main query (note suggest to have a initial Id to limit the size of the result)

let
 fn = (initId) =>
  let
   token = fnGetTokenHeader(),
   I = if initId = "" then fnGetMaxId(token){0}[maxid] else initId,
   endPoint = "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={this is the account}&max_id=" & I & "&count=200",
   records = fnGetResults(token, endPoint),
   tbl = Table.FromList(records, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
   tblMaxIds = Table.ExpandRecordColumn(tbl, "Column1", {"id_str"}, {"maxid"}),
   minId = try List.Min(tblMaxIds[maxid]) otherwise null,
   recordSet = [records = records, nextId = minId]
  in
   recordSet,
 resultSet = List.Generate(
                    () => fn(""),
                    each Number.FromText(_[nextId]) >= 1000000000000000000, //an max id you want to put to limit result size, you should get a value by calling getMaxId function, otherwise it could return too many lists
     // each _[nextId] <> null,
                    each fn(_[nextId]),
                    each [nextId = _[nextId], records = _[records]]
                    )
in
    resultSet



No comments :

Post a Comment