# 自定义技能设备端实现说明

# 设备端自定义技能实现逻辑

iFLYOS 云端获取上传的语义或文本请求后进行语义理解和识别,前置拦截器拦截自定义技能语料并下发自定义指令。iFLYOS 云端下发:

{"iflyos_meta": {
	"is_last": true,
	"trace_id": "86e626e3a9a807d20bde1f66b97cf6cc",
	"request_id": "manual_4b0ff606-3f95-416a-b26b-f78f6a48af0e"
 },
"iflyos_responses": [
	{
		"header": {
			"name": "audio_player.audio_out"
		},
		"payload": {
			"type": "TTS",
			"resource_id": "Gdnj7mLrz4scKZXy6QqzR8vkv9YeieugZ6wm2DyNAv5D0VIKgACJOqyIXaEdGIRN",
			"behavior": "SERIAL",
			"metadata": {
				"text": "正在为您打开设置"
			},
			"url": "http://tts.iflyos.cn/live/79c3738c6e7e2e3abc332e4762dfc6671ec95735.mp3",
			"secure_url": "https://tts.iflyos.cn/live/79c3738c6e7e2e3abc332e4762dfc6671ec95735.mp3"
		}
	}, {
		"header": {
			"name": "interceptor.custom"
		},
		"payload": {
			"headerName": "system.open.setting",
			"data": {
				"appName": "设置"
			}
		}
	}
]
}

解析自定义技能指令 “system.open.setting” 并执行打开设置操作,通过自定义技能执行打开第三方应用技能完成。

# Android 设备端基于 EVS SDK 的 ShowCore 自定义技能实现

设备端 ShowCore 代码需要解析自定义技能下发的控制指令并执行相应操作,示例中“打开第三方应用”自定义技能识别“打开相机”的语音请求后下发控制指令,设备端 ShowCore 需要解析 iFLYOS 云端下发的 “system.open.setting” 指令并执行打设置操作。

# 新建 CustomInterceptor 类

继承 interceptor 自定义拦截器,解析自定义指令,如下图所示。文本提供了 CustomInterceptor 类的示例代码;新建 custom 目录并 CustomInterceptor 放置在 custom 目录下,完整路径如下:

com.iflytek.cyber.iot.show.core.custom.CustomInterceptor

# 启动 CustomInterceptor 自定义拦截器类添加代码

override fun overrideInterceptor(): Interceptor? {  	
	return CustomInterceptor(this)  
}

新增 CustomInterceptor 类如图示:

启动自定义拦截器类如图所示:

自定义拦截器示例 CustomInterceptor 代码:

class CustomInterceptor(context: Context) : Interceptor() {

private val TAG = "CustomInterceptor"
private var context: Context? = null

init {
    Log.d(TAG, "init")
    this.context = context
}

override fun onResponse(name: String, payload: JSONObject) {
    Log.d(TAG, "name = $name")
    Log.d(TAG, "payload = $payload")
    payload?.let { json ->
        if (json.contains("headerName")) {
            val headerName = json.get("headerName")
            val data = json.getJSONObject("data")
            Log.d(TAG, "headerName = $headerName")
            Log.d(TAG, "data = $data")
            when (headerName) {
                "system.open.camera" -> {
                    Log.d(TAG, "system.open.camera")
                    val intent = Intent(MediaStore.ACTION_IMAGE_CAPTURE)
                    intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
                    context?.startActivity(intent)
                }
                "system.open.gallery" -> {
                    val intent = Intent(Intent.ACTION_VIEW)
                    intent.type = "image/*"
                    intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
                    context?.startActivity(intent)
                }
                "system.open.setting" -> {
                    val intent = Intent()
                    intent.setClassName("com.android.settings", "com.android.settings.Settings")
                    intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
                    context?.startActivity(intent)
                }
            }
        }
    }
}
}

# 测试自定义技能

重新将修改后的 ShowCore 项目运行到 Android 设备上,唤醒后分别测试自定义技能语料:"打开设置"、"打开图库"、"打开相机",ShowCore 控制台 logcat 信息中可以看到解析成功的指令信息,如图4-3所示:

# linux 平台自定义技能实现

以基于 EVS WebSocket 的 Shell+Python Demo 为示例描述 Linux 平台自定义技能实现方法。

# 设备授权

环境要求:

1.Shell 环境 支持 curl

2.Shell 环境 支持 jq

如果不支持,请下载交叉编译至目标平台

在目标机器的用户启动脚本.bashrc 或者.profile 中添加如下脚本代码,让每次开机时执行,进行机器授权信息的更新

if [ -e Implicit_authorization.sh ];then
	./Implicit_authorization.sh
fi

Implicit_authorization.sh 与~/.bashrc 或者~/.profile 放置在同级目录, 给予可执行权限,并将默认shell修改至当前的shell

Implicit_authorization.sh

#!/bin/bash

echo "################################################"
echo -n "#   " && date
echo "#               Auth Start!                     "
echo "################################################"
echo ""
echo ""
###############################################
client_id=""
device_id=""   

if [ $# -eq 2 ];then
	client_id=$1
	device_id=$2
fi

###############################################
access_token_file="access_token.json"
## 0. check Whether Authorization is overdue    

if [ ! $client_id ] && [ ! $device_id ];then
	echo "cliend_id or device_id is null!"
	exit
fi

echo "client_id:"$client_id
echo "device_id:"$device_id  

if [ -e "$access_token_file" ];then
	access_token_local_info=$(cat $access_token_file)
	ret=$(( $(echo $access_token_local_info | jq ".created_at") - $(date +%s) + $(echo $access_token_local_info | jq ".expires_in")))
	echo "################################################"
	echo -n "#   local time:" && date "+%Y-%m-%d %H:%M:%S"
	echo -n "#   created_at:" && date -d "@$(echo $access_token_local_info | jq ".created_at")" "+%Y-%m-%d %H:%M:%S"
	echo -n "#   expired_in:" && echo $(($(echo $access_token_local_info | jq ".expires_in")/3600/24)) "天" 
	echo -n "#   The remaining number of days:" && echo $((ret/3600/24)) "天"
	echo "################################################"
	if [ $(($ret-24*60*60)) -gt 0 ];then
		echo "Congratunations, access token is Within the validity period!"
		exit
	else
		echo "Less than one day remaining, now fresh access token"    

		url_refresh="https://auth.iflyos.cn/oauth/ivs/token"
		header_refresh="content-type:application/json"
		params_refresh="{\"grant_type\": \"refresh_token\",\"refresh_token\":$(echo $access_token_local_info | jq ".refresh_token")}"
		result_refresh=$(curl -X POST $url_refresh -H $header_refresh -d "$params_refresh")

		echo $result_refresh | jq

		if [ $(echo $result_refresh | grep access_token) ];then
		echo $result_refresh | jq ". | {token_type: .token_type, refresh_token:.refresh_token, expires_in: .expires_in, created_at:.created_at, access_token:.access_token, device_id:${device_id} }" > $access_token_file
			echo "Congratunations, access token is refreshed OK!"
			exit
		fi	
	fi
fi    

## 1. post new accesscode 
url="https://auth.iflyos.cn/oauth/ivs/device_code"
header="'content-type:application/x-www-form-urlencoded'"
scope_data="scope_data={\"user_ivs_all\": {\"device_id\":  \"$device_id\"}}"
scope_data_urlencode=$(echo -n $scope_data | sed -e 's/\"/%22/g' -e 's/\:/%3A/g' -e 's/{/%7B/g' -e 's/\}/%7D/g' -e 's/ /%20/g' )
params="client_id=${client_id}&scope=user_ivs_all&$scope_data_urlencode"    

#echo $params

commands="curl -X POST $url -H $header -d $params"
#echo $commands
echo "################################################"
echo "#      1. request device_code user_code         "
echo -n "#   " && date
echo "################################################"
ret=$(echo $(${commands} 2>/dev/null))

#ret="{\"verification_uri\":\"https://auth.iflyos.cn/oauth/device\",\"user_code\":\"129523\",\"interval\":5,\"expires_in\":120,\"device_code\":\"AB-YIIk52SnJS1awoIJJS4g2JYWawIaHgVa77a_g6OEKIYAeXNCTpuztHBaIV-RD\"}"
#echo $ret
echo $ret | jq

user_code=$(echo $ret | jq '.user_code')
device_code=$(echo $ret | jq '.device_code')
verification_uri=$(echo $ret | jq '.verification_uri')

#echo $ret | jq '.user_code'
#echo $user_code   

#######################################################################################################
## 3. query result
url_query="https://auth.iflyos.cn/oauth/ivs/token"
header_query="content-type:application/json"
params_query="{\"client_id\": \"$client_id\", \"grant_type\":\"urn:ietf:params:oauth:grant-type:device_code\", \"device_code\":$device_code}"    

{
	sleep 1
	echo "################################################"
	echo "#      3. query result                          "
	echo -n "#   " && date
	echo "################################################"
	for i in `seq 30`
	do
		{
			sleep 1
			result_query=$(curl -X POST $url_query -H $header_query -d "$params_query" 2>/dev/null)
			#echo "result_query:"$result_query 
			echo $result_query | jq
			#echo "i:$i"
			if [ $(echo $result_query | grep access_token) ];then
				echo $result_query | jq ". | {token_type: .token_type, refresh_token:.refresh_token, expires_in: .expires_in, created_at:.created_at, access_token:.access_token, device_id:${device_id} }" > $access_token_file
				exit
			fi
		}
	done
}&

#######################################################################################################
## 2. Implicit authorization 

url_implicit_auth="https://api.iflyos.cn/thirdparty/watch/auth"
header_implicit_auth="content-type:application/json"
#params_second="{\"thirdparty_id\":\"$client_id\",\"user_code\":\"129523\"}"
params_second="{\"thirdparty_id\":\"$device_id\",\"user_code\":$user_code}"   

echo "#########################################"
echo "#      2. Implicit authorization         "
echo "#               start                    "
echo -n "#   " && date 
echo "#########################################"    

#result_implicit_auth=$(curl -X POST $url_implicit_auth -H $header_implicit_auth -d "$params_second")
result_implicit_auth=$(curl -X POST $url_implicit_auth -H $header_implicit_auth -d "$params_second" 2>/dev/null)    

echo "#########################################"
echo "#      2. Implicit authorization         "
echo "#              result                    "
echo -n "#      " && echo $result_implicit_auth
echo "#########################################"
#$commands_user_auth
#${echo $commands_user_auth}
#result_implicit_auth=$(echo $(${commands_user_auth} ))
#echo $result_implicit_auth | jq

wait  ##等待所有子后台进程结束
echo "############################################"
echo -n "#   " && date
echo "#               Auth End!                   "
echo "############################################"

授权成功后会在当前目录生成access_token.json

{
  "token_type": "bearer",
  "refresh_token": "THIRDPARTY_f71L3mu2K2aOubwORiqaRRS7M9vlG1zBtIKgVoUhAyERLOSP6NlFwR9lzGsymC7I",
  "expires_in": 8640000,
  "created_at": 1583155505,
  "access_token": "THIRDPARTY_oHNaTIK8KP9A-zcKp4lUtS43r4m1TgR5YPDZ_e_EHKdCOhLDNhfsQucO0hgr0g-y",
  "device_id": xxx
}

# 建立 EVS 连接

设备端通过 EVS 协议与 iFLYOS 云端建立 WebSocket 连接:

self.ws = create_connection(base_url + "?token=" + access_token  + "&device_id=" + device_id)

# 设备端发送语音/文本请求

向云端发送语音请求:

def sendRequest(self):
        data = {'iflyos_header':{'authorization':'Bearer '  + access_token,
         'device':{'device_id': device_id, 'ip': '192.168.1.1',
         'location':{'latitude':81.56481, 'longitude':22.36549},
         'platform':{'name':'Linux', 'version': '8.1'}
                                          }},
    'iflyos_context': { 'system': { 'version' : '1.0'},
      'recognizer': { 'version' : '1.0'},
      'speaker': { 'version': '1.0', 'volume': 10, 'type': 'percent'},
             'audio_player': { 'version': '1.0', 'playback':{'state':'PLAYING'}},
                    },
     'iflyos_request': { 'header': {
                   'name': 'recognizer.audio_in',
                   'request_id': '6666'},
                   'payload': { 'reply_key': 'xxxxxx',
                      'enable_vad': 'true',
                      'profile': 'CLOSE_TALK',
                      'format': 'AUDIO_L16_RATE_16000_CHANNELS_1',
                      'iflyos_wake_up': {
                        'score': 666,
                        'start_index_in_samples': 50,
                        'end_index_in_samples': 150,
                        'word': '蓝小飞',
                        'prompt': '我在'
                                                  }}
                                                  }
                }
        #print str(data)
        jsondata = json.dumps(data)
        print jsondata
        self.ws.send(str(jsondata))

向云端发送音频:

def send(self, file_path):
    file_object = open(file_path, 'rb')
    try:
    index = 1
        while True:
        chunk = file_object.read(640)
            #chunk = file_object.read(1280)
            if not chunk:
            break
            #.send_binary
            if index == 1:
            print type(chunk)
            self.ws.send_binary(chunk)
            index += 1
            time.sleep(0.04)
     finally:
     #print str(index) + ", read len:" + str(len(chunk)) + ", file tell:" + str(file_object.tell())
     file_object.close()
         self.ws.send(bytes(end_tag))
         print "send end tag success"

开发者也可以向云端发送文本请求来实现,实现示例如下:

向云端发文本请求:

def sendTextRequest(self, text):
        data = {'iflyos_header': {'authorization': 'Bearer ' + access_token,
             'device': {'device_id': device_id, 'ip': '192.168.1.1',
             'location': {'latitude': 81.56481, 'longitude': 22.36549},
              'platform': {'name': 'Linux', 'version': '8.1'}
            }},
       'iflyos_context': {'system': {'version': '1.0'},
         'recognizer': {'version': '1.0'},
       'speaker': {'version': '1.0', 'volume': 10, 'type': 'percent'},
          'audio_player': {'version': '1.0', 'playback': {'state': 'PLAYING'}},
            },
            'iflyos_request': {'header': {
                    'name': 'recognizer.text_in',
                    'request_id': '6666'},
            'payload': {'reply_key': 'xxxxxx',
                        'query': text,
                        'with_tts': True,
                        'iflyos_wake_up': {
                            'score': 666,
                            'start_index_in_samples': 50,
                            'end_index_in_samples': 150,
                            'word': '蓝小飞',
                            'prompt': '我在'
                        }}
        }
        }
        # print str(data)
        jsondata = json.dumps(data)
        print jsondata
        self.ws.send(str(jsondata))

# Linux 平台解析 iFLYOS 云端下发自定义技能指令并执行操作

以打开微信为示例,Linux 平台解析 “system.open.wechat” 自定义技能指令并调用 Shell 脚本进行应用窗体和进程的管理,示例代码如下:

def parse_result(self, result):
    print u'%s' % result
    self.tparse = threading.Thread(target=self.parse_result_subThread, args=(result, ))
    self.tparse.start()
def parse_result_subThread(self, result):
    result_dict = json.loads(result)
    try:
       for i in result_dict.get('iflyos_responses'):
           if i.get('header').get('name') == 'audio_player.audio_out':
               url = i.get('payload').get('secure_url')
               # 播放TTS提示音
               os.system('curl -L %s | ffplay -nodisp -autoexit - &' % url)
           elif i.get('header').get('name') == 'interceptor.custom':
               intent = i.get('payload').get('headerName')
               if intent == 'system.open.wechat':
                   slots = i.get('payload').get('data').get('contact')
                   if slots == u'微信':
                      # 打开微信
                      os.system('./shell/system_open %s &'%'wechat')
     except Exception as err:
     print 'parse_result err: %s!'%err