signed

QiShunwang

“诚信为本、客户至上”

抖音爬虫最新方法

2021/4/26 13:56:17   来源:

最近有需求需要爬取抖音的一些数据,网上的一些方法都比较老,由于抖音升级较快已经不太实用了,所以只能自己解决了。

本次是对最新版本抖音app(13.6.0版本)进行的数据的抓取。
主要是通过脱壳、frida的逆向开发,在java层和Native层请求参数逆向还原,以及ida动态调试so文件破解请求参数和加密算法实现的爬取。

目前已经能爬取到抖音的个人中心、首页推荐视频列表、关注视频列表、同城视频列表、视频详情、用户粉丝列表、用户关注列表、用户作品列表、用户喜欢列表、用户权限、抖音热榜、明星榜、直播榜、音乐榜等。

一、个人中心数据(以明星万茜为例)

{
    "user":{
        "custom_verify":"演员万茜",
        "followers_detail":[
            {
                "name":"抖音",
                "icon":"http://p3.pstatp.com/origin/50ec00079b64de2050dc",
                "fans_count":3117098,
                "open_url":"snssdk1128://user/profile/2559257897875871?",
                "apple_id":"1142110895",
                "download_url":"https://d.douyin.com/JsvN/",
                "package_name":"com.ss.android.ugc.aweme",
                "app_name":"aweme"
            },
            {
                "fans_count":837857,
                "open_url":"snssdk143://profile?uid=54131370834",
                "apple_id":"529092160",
                "download_url":"https://d.toutiao.com/YjjY/",
                "package_name":"com.ss.android.article.news",
                "app_name":"news_article",
                "name":"头条",
                "icon":"http://p3.pstatp.com/origin/50ed00079a1b6b8d1fb1"
            },
            {
                "apple_id":"1086047750",
                "download_url":"http://d.huoshanzhibo.com/eFvB/",
                "package_name":"com.ss.android.ugc.live",
                "app_name":"live_stream",
                "name":"抖音火山版",
                "icon":"http://p3.pstatp.com/origin/2ea5c000abe106154adef",
                "fans_count":0,
                "open_url":"snssdk1112://profile?id=0"
            }
        ],
        "commerce_info":{
            "offline_info_list":[

            ],
            "challenge_list":[

            ],
            "task_list":null,
            "head_image_list":null,
            "smart_phone_list":null
        },
        "watch_status":false,
        "is_activity_user":false,
        "is_block":false,
        "with_commerce_entry":true,
        "profile_tab_type":0,
        "sec_uid":"MS4wLjABAAAA4_lIixBk6FAtLaq52ONCBDAqOd-qQtLAgMgGpjVEd0mLgZ8CM8nZA4TImrHTGnah",
        "enterprise_user_info":"{"commerce_info":{"offline_info_list":[],"challenge_list":[],"task_list":null,"head_image_list":null,"smart_phone_list":null},"homepage_bottom_toast":[],"permissions":[{"Id":4,"Key":"LiveShop","Name":"直播电商","AppId":1128,"Status":1,"Extra":null,"Customization":null,"Parent":0,"Actions":null},{"Id":5,"Key":"UserShop","Name":"个人橱窗","AppId":1128,"Status":1,"Extra":null,"Customization":null,"Parent":0,"Actions":null},{"Id":3,"Key":"ItemShop","Name":"视频电商","AppId":1128,"Status":1,"Extra":null,"Customization":null,"Parent":0,"Actions":null}]}",
        "follow_status":1,
        "sync_to_toutiao":1,
        "secret":0,
        "ins_id":"",
        "dongtai_count":28,
        "life_story_block":{
            "life_story_block":false
        },
        "tab_settings":{
            "private_tab":{
                "show_private_tab":false,
                "private_tab_style":1
            }
        },
        "message_chat_entry":true,
        "nickname":"万茜",
        "aweme_count":28,
        "school_name":"",
        "twitter_id":"",
        "with_fusion_shop_entry":true,
        "mplatform_followers_count":3954955,
        "country":"中国",
        "following_count":7,
        "room_id":0,
        "star_billboard_rank":16,
        "is_effect_artist":false,
        "is_star":true,
        "is_mix_user":false,
        "signature":"演员万茜",
        "show_favorite_list":true,
        "forward_count":0,
        "uid":"2559257897875871",
        "twitter_name":"",
        "follower_status":0,
        "district":"滨江",
        "video_cover":{

        },
        "original_musician":{
            "music_used_count":0,
            "digg_count":0,
            "music_count":0
        },
        "r_fans_group_info":{

        },
        "birthday_hide_level":1,
        "city":"杭州",
        "follower_count":3117098,
        "youtube_channel_title":"",
        "short_id":"0",
        "enterprise_verify_reason":"",
        "youtube_channel_id":"",
        "iso_country_code":"",
        "with_commerce_enterprise_tab_entry":false,
        "is_blocked":false,
        "display_wvalantine_activity_entry":true,
        "location":"杭州",
        "verification_type":1,
        "unique_id":"dyzd1sj2p8p3",
        "apple_account":0,
        }
    },
    "extra":{
        "now":1605952430000,
        "fatal_item_ids":[

        ],
        "logid":"2020112117535001020208903058B1B831"
    },
    "log_pb":{
        "impr_id":"2020112117535001020208903058B1B831"
    },
    "status_code":0
}

 

二、关注列表

三、视频评论列表数据

四、作品详情数据(数据较多,只列举了评论、点赞、播放数量等部分数据)

其他数据就不依次列举了。