{ localUrl: '../page/Sovereign.html', arbitalUrl: 'https://arbital.com/p/Sovereign', rawJsonUrl: '../raw/1g3.json', likeableId: '413', likeableType: 'page', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], pageId: 'Sovereign', edit: '9', editSummary: '', prevEdit: '8', currentEdit: '9', wasPublished: 'true', type: 'wiki', title: 'Autonomous AGI', clickbait: 'The hardest possible class of Friendly AI to build, with the least moral hazard; an AI intended to neither require nor accept further direction.', textLength: '1906', alias: 'Sovereign', externalUrl: '', sortChildrenBy: 'likes', hasVote: 'false', voteType: '', votesAnonymous: 'false', editCreatorId: 'EliezerYudkowsky', editCreatedAt: '2016-06-06 22:39:50', pageCreatorId: 'EliezerYudkowsky', pageCreatedAt: '2015-12-28 22:03:22', seeDomainId: '0', editDomainId: 'EliezerYudkowsky', submitToDomainId: '0', isAutosave: 'false', isSnapshot: 'false', isLiveEdit: 'true', isMinorEdit: 'false', indirectTeacher: 'false', todoCount: '0', isEditorComment: 'false', isApprovedComment: 'true', isResolved: 'false', snapshotText: '', anchorContext: '', anchorText: '', anchorOffset: '0', mergedInto: '', isDeleted: 'false', viewCount: '396', text: 'An autonomous or self-directed [2c advanced agent], a machine intelligence which acts in the real world in pursuit of its preferences without further user intervention or steering. In [18k Bostrom]'s [1g0 typology] of [2c advanced agents], this is a "Sovereign" and distinguished from a "[6w Genie]" or an "[6x Oracle]". ("Sovereign" in this sense means self-sovereign, and is not to be confused with the concept of a [Bostromian singleton](http://www.nickbostrom.com/fut/singleton.html) or any particular kind of social governance.)\n\nUsually, when we say "Sovereign" or "self-directed", we'll be talking about a supposedly [5s aligned] AI that acts autonomously *by design*. Failure to solve the alignment problem probably means the resulting AI is self-directed-by-default.\n\nTrying to construct an autonomous Friendly AI suggests that we trust the AI more than the [9r programmers] in any conflict between them, and we're okay with removing all constraints and off-switches except those the agent voluntarily takes upon itself.\n\nA *successfully* aligned autonomous AGI would carry the least [2sb moral hazard] of any scenario, since it hands off steering to some fixed [5f preference framework] or objective that the programmers can no longer modify. Nonetheless, being really really really that sure, not just getting it right but *knowing* we've gotten it right, seems like a large enough problem that perhaps we shouldn't be trying to build this class of AI *for our first try*, and should first target a [6w Task AGI] instead, or something else involving ongoing user steering.\n\nAn autonomous [41l superintelligence] would be the most difficult possible class of AGI to [5s align], requiring [41k total alignment]. [3c5 Coherent extrapolated volition] is a proposed alignment target for an autonomous superintelligence, but again, probably not something we should attempt to do on our first try.', metaText: '', isTextLoaded: 'true', isSubscribedToDiscussion: 'false', isSubscribedToUser: 'false', isSubscribedAsMaintainer: 'false', discussionSubscriberCount: '1', maintainerCount: '1', userSubscriberCount: '0', lastVisit: '2016-02-13 01:10:29', hasDraft: 'false', votes: [], voteSummary: 'null', muVoteSummary: '0', voteScaling: '0', currentUserVote: '-2', voteCount: '0', lockedVoteType: '', maxEditEver: '0', redLinkCount: '0', lockedBy: '', lockedUntil: '', nextPageId: '', prevPageId: '', usedAsMastery: 'false', proposalEditNum: '0', permissions: { edit: { has: 'false', reason: 'You don't have domain permission to edit this page' }, proposeEdit: { has: 'true', reason: '' }, delete: { has: 'false', reason: 'You don't have domain permission to delete this page' }, comment: { has: 'false', reason: 'You can't comment in this domain because you are not a member' }, proposeComment: { has: 'true', reason: '' } }, summaries: {}, creatorIds: [ 'EliezerYudkowsky' ], childIds: [], parentIds: [ 'AGI_typology' ], commentIds: [ '1gk', '1gl' ], questionIds: [], tagIds: [], relatedIds: [ 'cev' ], markIds: [], explanations: [], learnMore: [], requirements: [], subjects: [], lenses: [], lensParentId: '', pathPages: [], learnMoreTaughtMap: {}, learnMoreCoveredMap: {}, learnMoreRequiredMap: {}, editHistory: {}, domainSubmissions: {}, answers: [], answerCount: '0', commentCount: '0', newCommentCount: '0', linkedMarkCount: '0', changeLogs: [ { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '11845', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '0', type: 'deleteTag', createdAt: '2016-06-06 22:40:02', auxPageId: 'stub_meta_tag', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '11843', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '9', type: 'newEdit', createdAt: '2016-06-06 22:39:50', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '11842', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '8', type: 'newEdit', createdAt: '2016-06-06 21:49:32', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '11841', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '7', type: 'newEdit', createdAt: '2016-06-06 21:48:13', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '11840', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '6', type: 'newEdit', createdAt: '2016-06-06 21:45:33', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '9420', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '5', type: 'newUsedAsTag', createdAt: '2016-04-27 02:19:39', auxPageId: 'cev', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '4765', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '5', type: 'newEdit', createdAt: '2015-12-30 00:40:18', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '4764', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '4', type: 'newEdit', createdAt: '2015-12-30 00:39:24', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '4589', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '3', type: 'newEdit', createdAt: '2015-12-28 22:07:06', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '4588', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '2', type: 'newEdit', createdAt: '2015-12-28 22:06:51', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '4586', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '1', type: 'newEdit', createdAt: '2015-12-28 22:03:22', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '4576', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '0', type: 'newTag', createdAt: '2015-12-28 21:47:34', auxPageId: 'stub_meta_tag', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '4574', pageId: 'Sovereign', userId: 'EliezerYudkowsky', edit: '0', type: 'newParent', createdAt: '2015-12-28 21:46:26', auxPageId: 'AGI_typology', oldSettingsValue: '', newSettingsValue: '' } ], feedSubmissions: [], searchStrings: {}, hasChildren: 'false', hasParents: 'true', redAliases: {}, improvementTagIds: [], nonMetaTagIds: [], todos: [], slowDownMap: 'null', speedUpMap: 'null', arcPageIds: 'null', contentRequests: {} }